level=info ts=2026-06-30T17:58:23.499181279Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-30T17:58:23.499212526Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-30T17:58:23.500466154Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-06-30T17:58:23.501526424Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-2cd94bba
level=info ts=2026-06-30T17:58:23.501757875Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-06-30T17:58:23.506847677Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-06-30T17:58:23.508241404Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-30T17:58:23.508272162Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T17:58:23.508300086Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.536566ms
level=error ts=2026-06-30T17:58:23.508317122Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-30T17:58:23.508330222Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-06-30T17:58:23.508548667Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-06-30T17:58:23.508594366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T17:58:23.509110211Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-06-30T17:58:23.509151233Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-06-30T17:58:23.509177686Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
ts=2026-06-30T17:58:23.511180857Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T17:58:23.511194656Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-30T17:58:23.524511979Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-30T17:58:23.524545338Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-30T17:58:23.524566758Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-30T17:58:23.524831889Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-06-30T17:58:23.524846917Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-30T17:58:23.524878698Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-30T17:58:23.524913297Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-30T17:58:23.525045594Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-06-30T17:58:23.525061716Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-06-30T17:58:23.525074633Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T17:58:23.52512985Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=29.527µs success=true
level=info ts=2026-06-30T17:58:23.525144139Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-06-30T17:58:23.525212908Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=153.283µs errors=false
level=info ts=2026-06-30T17:58:23.525220192Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-06-30T17:58:23.525597813Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=536.095µs errors=false
level=info ts=2026-06-30T17:58:23.525614756Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-06-30T17:58:23.525621579Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=560.929µs
level=info ts=2026-06-30T17:58:23.52564153Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-06-30T17:58:23.525672522Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-06-30T17:58:23.525665276Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=1.231938092s
level=info ts=2026-06-30T17:58:23.525694478Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-06-30T17:58:23.525763149Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-06-30T17:58:23.525764745Z caller=loki.go:615 msg="Loki started" startup_time=86.785756ms
level=info ts=2026-06-30T17:58:23.525791898Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
ts=2026-06-30T17:58:25.064404909Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T17:58:25.064427367Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-30T17:58:28.061389583Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T17:58:28.061413634Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T17:58:28.526347138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:58:33.526697202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-06-30T17:58:34.250311424Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T17:58:34.250334851Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T17:58:38.526742846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:58:43.526597888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-06-30T17:58:43.571927172Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-30T17:58:43.571950657Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-30T17:58:48.526308287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T17:58:53.525877712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T17:58:53.525939848Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-06-30T17:58:53.52595105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T17:58:53.52598317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:58:58.526456233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T17:59:00.294334359Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=4 elapsed_time=36.785997539s
level=warn ts=2026-06-30T17:59:03.526272463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:08.526731607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:13.526339332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:18.526626113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T17:59:23.508722295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T17:59:23.52582606Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T17:59:23.525862972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T17:59:23.525910239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:28.526510867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:33.526705955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:38.526831807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T17:59:40.847858394Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEDB310EAD827, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T17:59:43.526776097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T17:59:48.526819062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T17:59:53.526215139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T17:59:53.526253703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T17:59:53.526260933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T17:59:56.458333793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T17:59:56.473483364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDB6B4432378, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T17:59:58.526736438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:03.526787683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:08.526051487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:13.526562573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:00:14.353548792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:00:14.364512186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDBADEA8596F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:00:18.525925451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:00:23.508676104Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:00:23.526395633Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:00:23.526432054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:00:23.526450933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:28.526709613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:33.526545821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:38.525934419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:43.526223384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:00:47.34982585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:00:47.361543878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDC28D6E0E56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:00:48.526139373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:00:53.526206303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:00:53.526243256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:00:53.526261436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:00:58.526180779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:03.525955795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:08.526473385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:13.526428723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:18.526152756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:01:23.508964913Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:01:23.526373408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:01:23.526405617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:01:23.526426142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:28.52690474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:33.526068745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:01:35.186302472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:01:35.195926672Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDCDB09603E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:01:38.526705073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:43.526144987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:48.526278431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:01:53.526480626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:01:53.526518921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:01:53.526521123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:01:58.526477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:03.526165622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:08.526433823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:13.526867332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:02:16.75647557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:02:16.771692272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDD75EB20091, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:02:18.526943105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:02:23.509665913Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:02:23.525909268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:02:23.525948392Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:02:23.525941537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:28.52625702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:33.526782713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:02:37.735561113Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEDDC403E56AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:02:38.5264885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:43.52661735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:48.526267462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:02:53.526364499Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:02:53.526405953Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:02:53.526430457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:02:58.526807752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:03:01.926232368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:03:01.937975802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDE1E2D116B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:03:03.525949633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:08.526236911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:13.526811402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:18.526168457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:03:23.508796475Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:03:23.525726828Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:03:23.525866189Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-06-30T18:03:23.525875041Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:03:23.52589216Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.092µs success=true
level=info ts=2026-06-30T18:03:23.525901183Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:03:23.52592735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:03:23.525936068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:28.52603525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:33.526477016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:38.526020766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:43.526761562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:03:46.216321587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:03:46.229514907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDEC32CB4EE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:03:48.526173291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:03:53.526348827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:03:53.526399987Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:03:53.5264227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:03:58.52675186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:03.526323554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:08.526632516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:13.526851365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:18.526698981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:04:23.508705298Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:04:23.526048572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:04:23.526077813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:04:23.526122068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:04:26.622088838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:04:26.632401297Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEDF59AFDB944, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:04:28.526390985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:33.52660969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:38.526032566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:43.526295653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:48.526091583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:04:53.525768187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:04:53.525802341Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:04:53.525882634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:04:58.526130738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:03.525896915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:08.526605087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:05:12.569481817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:05:12.583063649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE004DDC9D51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:05:13.52592821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:18.526608997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:05:23.50878517Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:05:23.52577407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:05:23.525810667Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:05:23.525894479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:28.526091975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:33.525982161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:38.526413647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:43.526822233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:48.526620944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:05:53.525756562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:05:53.52579736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:05:53.525876096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:05:58.526398935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:05:59.377018311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:05:59.389625046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE0B33C086DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:05:59.389650752Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:05:59.389675831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:05:59.398033534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE0B3442D6FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:06:03.526823526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:08.526561307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:06:10.483437933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:06:12.256381613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE0E32AB811C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:06:13.525978907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:18.52628212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:06:23.508743911Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:06:23.526611241Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:06:23.526648375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:06:23.52669583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:06:26.322596227Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEE1179185924, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:06:28.526908834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:33.526814664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:38.526213254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:06:40.510085982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:06:40.524241771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE14C7908498, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:06:43.526273155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:48.526136498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:06:53.52654412Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:06:53.52658235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:06:53.526582786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:06:58.526322485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:03.526786458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:08.526469503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:13.526686279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:18.526079313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:07:23.509234307Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:07:23.525969746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:07:23.526013148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:07:23.526009577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:07:28.001408261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:07:28.011716346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE1FD60B9FAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:07:28.526205113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:33.526776152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:38.526813686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:43.526886508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:07:48.526079902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:07:53.525858804Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:07:53.525896671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:07:53.52592803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:07:53.530532211Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-06-30T18:07:53.534333459Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008281955s
level=warn ts=2026-06-30T18:07:58.52632166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:03.526556147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:08.526035355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:13.526907137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:08:17.076892606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:08:17.089291276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE2B434AB428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:08:18.526744577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:08:23.508693148Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:08:23.526242737Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:08:23.526272337Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:08:23.526300474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:08:23.526326469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:08:23.526428876Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-06-30T18:08:23.526441401Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:08:23.526465509Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.521µs success=true
level=warn ts=2026-06-30T18:08:28.526684653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:33.526886627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:38.526496467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:43.52584161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:48.526090145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:08:53.526234532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:08:53.526276246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:08:53.526275154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:08:58.526631045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:03.526582989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:09:07.826101393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:09:07.838369289Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE37142D937F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:09:08.526104673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:13.526849307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:18.526218847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:09:23.508975821Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:09:23.525860859Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:09:23.525893225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:09:23.525926839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:28.526631334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:33.52655128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:38.52674467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:43.525881908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:48.526853484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:09:53.526480489Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:09:53.526527662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:09:53.526541208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:09:58.526193336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:10:02.146097674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:10:02.15861613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE43B9EA8C81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:10:03.526891814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:08.526773788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:13.526533505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:18.526833843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:10:23.509084118Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:10:23.525980007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:10:23.526015102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:10:23.526034046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:28.526859317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:33.526266627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:10:36.590010076Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEE4BBE31B2EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:10:38.526948304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:43.526712644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:48.526906048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:10:49.148959487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.8 MB" avg_comp="3.8 MB" total_uncomp="33 MB" avg_uncomp="33 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:10:49.161868113Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE4EAB85BF74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:10:53.526488519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:10:53.526525339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:10:53.526524703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:10:58.526908236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:03.526862425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:08.526286549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:13.526466924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:18.526722272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:11:23.508675527Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:11:23.52576612Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:11:23.525803226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:11:23.525882884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:28.526859815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:33.525907989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:38.525952847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:43.525909341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:11:45.206638633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:11:45.222047466Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE5BB8F93F76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:11:48.526924074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:11:53.526490643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:11:53.526533694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:11:53.526535167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:11:58.526193414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:03.525996119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:08.52633665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:13.525821988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:18.526443717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:12:23.508999901Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:12:23.525909589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:12:23.525911927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:12:23.525947061Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:12:28.526379879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:33.525892089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:12:36.652963807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:12:36.662630895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE67B31021FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:12:36.662656547Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:12:36.662683224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:12:36.671637244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE67B39D280E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:12:38.526403646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:43.526857683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:12:48.52620998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:12:53.092156806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:12:53.104649571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE6B8717F926, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:12:53.526461668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:12:53.526497619Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:12:53.526515175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:12:53.532100642Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-06-30T18:12:53.53388803Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007241998s
level=warn ts=2026-06-30T18:12:58.526690448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:03.52681105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:08.526332604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:13.526845777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:18.526438291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:13:18.812125884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:13:18.823626032Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE7184106915, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:13:23.50874751Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:13:23.525713662Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:13:23.525762584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:13:23.525795297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:13:23.525917464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:13:23.525972473Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-06-30T18:13:23.525984831Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:13:23.526008698Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.3µs success=true
level=warn ts=2026-06-30T18:13:28.526733028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:33.525816753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:38.526027153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:43.526153924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:48.526710639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:13:50.187537209Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEE78D18226B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:13:53.525777945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:13:53.525830679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:13:53.525893287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:13:58.526094006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:03.525824752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:14:05.77372726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:14:05.785606035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE7C7335F89C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:14:08.526735078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:13.526713268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:18.526432115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:14:23.509053926Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:14:23.525933662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:14:23.525966591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:14:23.525949886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:28.526761008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:33.526134534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:38.526891941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:43.526835027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:14:48.526193408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:14:53.526727068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:14:53.526735024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:14:53.526769903Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:14:56.091662571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:14:56.103580998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE882A65ABB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:14:58.525974203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:03.525966062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:08.526098142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:10.340046424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:15:10.347116168Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE8B7B5E158C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:15:13.526073031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:18.526176792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:23.50868826Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:15:23.525786373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:15:23.525824553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:15:23.525912897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:24.600747093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:15:24.609446394Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE8ECD7A4A6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:15:28.526500898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:33.526424095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:36.516335545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:15:36.529369473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE9193F39A12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:15:38.526563256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:43.526546664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:15:48.52715261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:53.526278886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:15:53.526318529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:15:53.526342301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:15:57.482387063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:15:57.491896213Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE96756C3B03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:15:58.526304586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:03.52654455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:08.526193888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:13.526337953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:18.52626129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:16:21.982252963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:16:21.99362319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEE9C29D78838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:16:23.508703032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:16:23.525751089Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:16:23.52578579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:16:23.525878554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:28.526025545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:33.526727666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:38.526811667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:43.526614912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:16:48.526745582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:16:53.526242282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:16:53.526284599Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:16:53.526299037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:16:55.929140169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:16:55.938755841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEA4111FC013, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:16:58.526738781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:03.526475151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:08.526283746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:17:10.02847016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:17:10.039317463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEA759967E7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:17:13.526388151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:18.52671915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:17:23.509127848Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:17:23.526602259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:17:23.526641791Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:17:23.526677359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:28.526756916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:33.526370126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:38.526759856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:17:43.283406575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:17:43.292529793Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEAF17A2BA48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:17:43.526750662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:17:48.526292464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:17:53.52626997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:17:53.526309327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:17:53.526349099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:17:53.534313932Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-06-30T18:17:53.536121375Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009950849s
level=error ts=2026-06-30T18:17:55.847298497Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEEB203F77E1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:17:58.526032033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:03.525916004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:18:05.941862521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:18:05.951789922Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEB45E3B1934, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:18:08.526070108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:13.526730284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:18.526968073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:18:23.509233743Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:18:23.526615204Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:18:23.526647244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:18:23.526655027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:18:23.526680532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:18:23.526751969Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-06-30T18:18:23.526760131Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:18:23.526778161Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.455µs success=true
level=warn ts=2026-06-30T18:18:28.526632773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:33.526847945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:18:34.021575074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:18:34.030908797Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEBAE7E05A73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:18:38.526713607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:43.5259494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:48.526086324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:18:53.525759185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:18:53.525799438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:18:53.52591971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:18:58.526852672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:03.526233909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:19:03.536315147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:19:03.549854123Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEC1C7564EFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:19:03.549878496Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:19:03.549907059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:19:03.559443622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEC1C7EBDB1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:19:08.52671868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:13.526634611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:18.525881351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:19:22.999039636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:19:23.011277449Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEC64F53BD2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:19:23.508764679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:19:23.52595149Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:19:23.525990434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:19:23.52598993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:28.526310185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:19:31.608198668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:19:31.616063153Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEC850389EC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:19:33.526547702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:38.526075204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:43.526070366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:19:45.774729608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:19:45.787346886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEECB9CE55148, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:19:48.52601483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:19:53.526584435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:19:53.526624839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:19:53.52663381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:19:58.52642614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:03.526517883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:08.525928211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:13.526730524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:18.526556573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:20:23.508816724Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:20:23.526376752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:20:23.526413688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:20:23.526437659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:28.52672193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:20:28.784338188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:20:28.791528944Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEED5A0248392, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:20:33.526465113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:38.525982843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:20:43.525981426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:20:44.581842049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:20:44.593744368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEED94E074153, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:20:48.526238901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:20:53.526694581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:20:53.526706549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:20:53.526732007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:20:58.526379482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:03.525917745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:08.526560463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:13.526242161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:18.526509112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:21:19.874521069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:21:19.881039911Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEE18550BA92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:21:23.509597545Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:21:23.526125024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:21:23.526142773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:21:23.526161298Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:21:28.526771487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:33.52669826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:21:37.447309882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:21:37.460215307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEE59D1CA7ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:21:38.526624132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:43.526789459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:48.526711637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:21:53.525774355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:21:53.525821269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:21:53.525906738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:21:58.526824892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:22:03.525851329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:07.43069966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:22:07.440776787Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEEC9818017F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:22:07.4408011Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:22:07.440826861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:22:07.446491482Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEEC98705388, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:22:08.526851147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:22:13.411923306Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEEEDFC0384A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:22:13.526738674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:18.314299357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:22:18.321119388Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEEF209CB96D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:22:18.52646429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:23.509381099Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:22:23.525845802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:22:23.525855403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:23.52588073Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:22:28.526558107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:30.462108154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:22:30.475773979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEF1F51477AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:22:33.526461319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:22:38.52685522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:22:43.526420254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:22:48.525959483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:50.150793531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:22:50.159165135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEF68A4E70E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:22:53.526610264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:22:53.526652453Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:22:53.526709382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:22:53.536607058Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-06-30T18:22:53.538314766Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011401359s
level=warn ts=2026-06-30T18:22:58.526569432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:03.526096714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:08.526493574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:11.008997677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:23:11.021073286Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEEFB65C55F5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:23:13.525809801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:18.526695062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:23.509160125Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:23:23.52622119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:23:23.526230495Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:23:23.526256825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:23:23.526338468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:23.526585342Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-06-30T18:23:23.526595657Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:23:23.526615412Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.379µs success=true
level=warn ts=2026-06-30T18:23:28.526264376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:33.526487481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:38.525871894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:40.697375554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:23:40.706803414Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF024F2DFD19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:23:43.526426132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:23:48.526684635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:53.526790021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:23:53.526833553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:23:53.526839698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:23:57.727799797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:23:57.738929845Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF06465FBDEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:23:58.526490371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:03.526628608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:08.5264603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:13.526740947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:18.526687845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:23.508672032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T18:24:23.526169612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:23.526173736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:24:23.52620398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:24:28.5260693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:33.526508598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:34.697055685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:24:34.705668669Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF0EE1C19A05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:24:38.525851601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:39.61831532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:24:39.627801412Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF1007261FAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:24:43.529780542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:24:48.526803765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:53.526700827Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:24:53.526714713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:24:53.52675003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:24:58.526597876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:03.52681498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:08.526513072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:13.526565379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:18.526333346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:25:22.732641256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:25:22.744904191Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF1A11213E89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:25:22.74492512Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:25:22.744951511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:25:22.753889628Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF1A11AB251F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:25:23.509270917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:25:23.525756822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:25:23.525789938Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:25:23.525826473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:25:23.844490264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:25:23.850611505Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF1A5308EA29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:25:28.526821942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:33.525883877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:38.525999526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:25:41.485935852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:25:41.495283324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF1E6EBCDFFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:25:43.526760773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:25:48.526638617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:25:53.526340136Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:25:53.526388919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:25:53.52640645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:25:58.526408056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:03.52651818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:08.5262517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:13.525978944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:26:16.800399644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:26:16.81161636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF26A7C11259, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:26:18.526756177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:26:19.770961257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:26:19.777643833Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF27588AB7A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:26:22.009069516Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEF27DD8E43E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:26:23.508847627Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:26:23.526574643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:26:23.526614563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:26:23.526628781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:28.526406716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:33.52661721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:38.52664465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:43.525901129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:48.526635835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:26:53.52595191Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:26:53.525993772Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:26:53.52602411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:26:58.526691757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:03.525878646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:08.526565339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:13.52581617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:27:15.380034217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:27:15.392777529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF344B737D90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:27:16.904184552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:27:16.91058871Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF34A5EDEC98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:27:18.526848479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:27:23.509264808Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:27:23.52676502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:27:23.526799333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:27:23.526809019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:28.526321379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:33.526784897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:38.526327179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:43.52591862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:27:48.526061112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:27:53.526398244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:27:53.5264333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:27:53.526461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:27:53.534329067Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-06-30T18:27:53.536071382Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009337853s
level=warn ts=2026-06-30T18:27:58.525851426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:02.532082243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:28:02.542664025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF3F45D07293, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:28:03.525947727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:06.44663584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:28:06.453784785Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF402EEFA5BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:28:08.526789535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:28:13.526632961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:28:18.526273994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:23.508686974Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:28:23.526670719Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:28:23.526684561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:28:23.526691981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:23.526709859Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:28:23.526824038Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-06-30T18:28:23.526833139Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:28:23.526850664Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.746µs success=true
level=warn ts=2026-06-30T18:28:28.526750541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:28:33.525951126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:28:38.526030579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:28:43.525888167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:43.98161511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:28:43.992176029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF48EC641E94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:28:48.526167485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:50.948645444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:28:50.955101977Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4A8B6CAB9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:28:50.955123888Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:28:50.955147639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:28:50.960535671Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4A8BC08E68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:28:53.526719859Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:28:53.526738644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:28:53.526758908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:28:58.526961821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:29:03.52681812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:03.5849781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:29:03.594763713Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4D7CCDBE4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:29:08.526012172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:10.94448523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="121 B" avg_comp="121 B" total_uncomp="112 kB" avg_uncomp="112 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:29:10.949863185Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4F3333213D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:29:11.244652205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:29:11.247277508Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4F44EDED2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:29:11.545549095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:29:11.548180275Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4F56DCF3C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:29:11.844931175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="101 B" avg_uncomp="101 B" idle=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:29:11.847383529Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4F68B3DCE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:29:12.145490928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:29:12.147986217Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF4F7A9E559C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:29:13.526754358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:29:18.526822336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:21.662482434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:29:21.664942962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF51B1DF5BF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:29:22.48496196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:29:22.487313508Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF51E2E4B12D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:29:23.509169882Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:29:23.526620327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:29:23.526636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:23.526654125Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:29:24.11579107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:29:24.118253146Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF52441A8F2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:29:27.062988249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:29:27.065510335Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF52F3C52B4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:29:27.518547708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:29:28.526864474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:30.203723069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:29:30.207906846Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF53AF1337A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:29:33.074232482Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5459EC2E66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:29:33.526471667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:38.202470315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:29:38.208950318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF558BF90711, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:29:38.366442065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:29:38.370936843Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5595A0DEF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:29:38.52651661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:38.666886891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="23 kB" avg_uncomp="23 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:29:38.669422987Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF55A76AE025, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:29:38.967480242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="451 B" avg_uncomp="451 B" idle=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:29:38.969915645Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF55B953EDFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:29:39.266969535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:29:39.269612179Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF55CB30F046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:29:39.566626703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:29:39.569090384Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF55DD0B3852, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:29:39.866685437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="32 kB" avg_uncomp="32 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:29:39.869323052Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF55EEEF96D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:29:40.167435424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:29:40.171437102Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5600DED6AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:29:40.46740421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:29:40.470106723Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5612BDC4B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:29:40.76731526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:29:40.769774723Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF56249CAADC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:29:41.066549737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="140 kB" avg_uncomp="140 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:29:41.069260347Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5636758B86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:29:43.526193187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:44.958114591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:29:44.960546029Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF571E668D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:29:48.025370222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:29:48.027910964Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF57D53AB693, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:29:48.526649477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:50.795754295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:29:50.798223371Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF587A5A6D0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:29:50.880301064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:29:50.882845562Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF587F64A6F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:29:50.91419389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:29:50.916499853Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF588167D64E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:29:50.961568049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:29:50.963890832Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF58843B0A97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:29:51.155484282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:29:51.158031443Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF588FCC0FB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:29:53.526782642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:29:53.526821596Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:29:53.526819778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:55.162975071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:29:55.165296028Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF597EA6DB1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:29:55.290503435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:29:55.293086135Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF598644722F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:29:56.16673193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:29:56.169315531Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF59BA7DA635, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:29:56.357847787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:29:56.360342968Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF59C5E056F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:29:58.526775932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:29:58.735239842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:29:58.737708321Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5A539405B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:29:59.71296609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:29:59.71551432Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5A8DDD1F1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:29:59.781642727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:29:59.784144837Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5A91F4228A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:30:01.02288701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:30:01.025496783Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5ADBF18215, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:30:03.525843042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:30:06.995961847Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEF5C3FD08208, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:30:08.52604766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:30:13.52647123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:16.547326276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:30:16.550635773Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5E79443484, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:30:18.526449237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:18.900235382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:30:18.908402186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5F05D6C424, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:30:19.204972774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:30:19.211277647Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5F17E70170, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:30:20.800545133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:30:20.803031101Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5F76C7AD9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:30:22.960462306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:30:22.962939624Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF5FF7858BBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:30:23.172275183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:30:23.174588475Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF600423E763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:30:23.508668872Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:30:23.526511471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:30:23.526547609Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:30:23.526589707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:26.029874682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:30:26.032262307Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF60AE778A14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:30:27.014916011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:30:27.017597697Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF60E9318AF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:30:27.765754647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:30:27.768155173Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6115EF1C62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:30:28.526711729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:29.564100244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:30:29.566550027Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6181203DA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:30:31.976037881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:30:31.980163053Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6210FD5AEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:30:33.52622162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:35.816684255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:30:35.823259115Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF62F60D4233, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:30:36.40816795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:30:36.410535713Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6319104E7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:30:38.526746003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:38.905867941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:30:38.908184287Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF63ADEDF7C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:30:43.526426577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:45.710896386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:30:45.713606968Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF654390FAF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:30:46.078402224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:30:46.080729994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF655973C080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:30:48.526474974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:50.26256677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:30:50.265019852Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6652D993CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:30:53.525864956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:30:53.525882175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:30:53.525914893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:30:58.526349748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:01.154227395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:31:01.158091498Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF68DC20A784, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:31:03.526301812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:04.390903977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:31:04.393379421Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF699CF7907E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:31:06.554620552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:31:06.562747853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6A1E45EF3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:31:07.284714296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:31:07.290825843Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6A49ABC005, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:31:07.57972687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:31:07.582107357Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6A5B082246, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:31:08.526825049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:10.438936405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:31:10.441587822Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6B0576781F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:31:13.525898829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:14.444646223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:31:14.446961337Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6BF435C87D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:31:15.157077347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:31:15.159460054Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6C1EADAF06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:31:18.526377307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:19.395587402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:31:19.398023703Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6D1B508112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:31:20.249480569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:31:20.251786153Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6D4E349159, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:31:21.10433131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:31:21.106858054Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6D812B8DE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:31:23.508885934Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:31:23.526368393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:31:23.526423948Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:31:23.526411906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:27.44348118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:31:27.446086263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6EFB04B592, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:31:27.852857781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:31:27.856997023Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6F138148F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:31:27.910312762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:31:27.914898323Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6F16F5172A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:31:28.526748755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:29.124694515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:31:29.127228379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6F5F38043A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:31:31.226374024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:31:31.228758311Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF6FDC7B8587, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:31:33.526566777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:31:38.526493396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:40.766412263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:31:40.768977354Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF72151ED0AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:31:43.526264035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:48.122282266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:31:48.129727317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF73CBDB58D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:31:48.129753341Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:31:48.129780719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:31:48.141312679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF73CC8E0363, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:31:48.526348436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:51.767013929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:31:51.77159369Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF74A4EEDE73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:31:52.037138476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:31:52.039719721Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF74B4E8647C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:31:53.526630427Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:31:53.526642156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:53.52667146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:31:53.901418892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:31:53.904199958Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7524091F68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:31:57.693616832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:31:57.696325823Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF76061243BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:31:58.52595074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:31:59.671558359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:31:59.674074454Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF767BF43A6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:31:59.800757938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:31:59.808605421Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7683F95755, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:32:03.526562393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:08.16617568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:32:08.172866548Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF787684292B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:32:08.416608454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:32:08.419244477Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7885343BB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:32:08.526644376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:08.773108633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:32:08.775527633Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF789A71E1C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:32:08.801741055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:32:08.803951019Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF789C236620, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:32:08.990622887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:32:08.99318007Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF78A76A5AAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:32:11.909963335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:32:11.914385076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF79558679A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:32:13.129775595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:32:13.134586025Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF799E4297E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:32:13.525925645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:14.814475694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:32:14.816831654Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7A0288930A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:32:18.526004055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:21.309965343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:32:21.312523843Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7B85B49A6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:32:21.86826664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:32:21.876374986Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7BA750B6B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:32:23.252783878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:32:23.257085649Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7BF99B6348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:32:23.509152213Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:32:23.526714322Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:32:23.526747477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:32:23.526765638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:32:28.526027274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:32:33.526200775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:32:38.526867919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:38.722391512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:32:38.724957365Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7F9390625A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:32:39.745721649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:32:39.748294744Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF7FD08ECDEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:32:42.334836289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:32:42.337465999Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF806AE237C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:32:43.525981455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:43.69947001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:32:43.701821889Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF80BC367A92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:32:43.818834928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:32:43.821192298Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF80C353DFAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:32:48.525867679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:52.936472221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:32:52.941026159Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF82E2E82BC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:32:53.526339789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:32:53.526399364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:32:53.526394869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:53.536225108Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-06-30T18:32:53.538173983Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011203586s
level=info ts=2026-06-30T18:32:54.915409764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:32:54.921968697Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8358FABF0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:32:56.072206819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:32:56.074640708Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF839DB02902, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:32:58.526692467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:32:58.852182062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:32:58.854626263Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF844363C290, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:32:59.493256407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:32:59.495729529Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF846999C035, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:33:03.526184448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:03.962585386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:33:03.965055128Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8573FDDD54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:33:05.599085164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:33:05.601691562Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF85D58A7539, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:33:05.871230387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:33:05.873768446Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF85E5C38111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:33:06.50653356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:33:06.514291066Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF860BF0555B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:33:08.526416226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:10.292179325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:33:10.296730348Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF86ED647A89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:33:12.294631685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:33:12.297024866Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF87649E6FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:33:13.526795769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:33:18.526203777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:18.73585877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:33:18.738643237Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF88E490B88E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:33:19.722814964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:33:19.725451181Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF891F631B27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:33:23.508860928Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:33:23.526316465Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:33:23.526310378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:33:23.526325785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:23.526342222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:33:23.526468667Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-06-30T18:33:23.526476803Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:33:23.526495945Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.404µs success=true
level=info ts=2026-06-30T18:33:24.011315501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:33:24.013736402Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8A1EFD189C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:33:28.526708974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:32.102376403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:33:32.104867563Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8C0141E8CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:33:32.415040765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:33:32.417653043Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8C13E640AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:33:33.526206571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:35.343787882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:33:35.3462865Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8CC2760188, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:33:36.31946165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:33:36.321949604Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8CFC9E5F24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:33:38.526561903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:33:43.525972379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:44.467855116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:33:44.472541142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8EE26E2855, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:33:46.167934646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:33:46.174042625Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8F47D8ED77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:33:46.973954768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:33:46.976294556Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8F77A98702, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:33:48.52605983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:48.763479162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:33:48.772453256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF8FE2B9135A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:33:51.141374529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:33:51.146206649Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9070347214, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:33:53.526727444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:33:53.526725974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:33:53.526768726Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-06-30T18:33:54.103293815Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEF9120797E68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:33:54.225568379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:33:54.227933935Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9127E50551, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:33:55.61530126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:33:55.617817623Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF917ABCC21F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:33:56.572965242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:33:56.575248022Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF91B3CE1703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:33:58.175717639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:33:58.178216187Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF92135953E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:33:58.526705851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:34:03.525911861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:34:08.526771531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:11.873546502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:34:11.877737928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9543E7F4A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:34:13.526269929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:15.024807973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:34:15.027218814Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF95FFA17BE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:34:15.230808781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:34:15.233268952Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF960BE8F0EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:34:18.316416512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:34:18.319112178Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF96C3D60EFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:34:18.526764044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:19.851943094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:34:19.854289707Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF971F582E5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:34:22.242167069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:34:22.244660624Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF97ADD20DF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:34:23.135979488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:34:23.138298516Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF97E3165209, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:34:23.508953731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:34:23.526289776Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:34:23.526306916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:23.526324129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:34:24.223106393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:34:24.225461822Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9823E35179, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:34:28.526139282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:29.170181515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:34:29.172659934Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF994AC3784A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:34:33.526799034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:35.739187383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:34:35.741993318Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9AD252BB38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:34:38.52687219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:38.656224703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:34:38.664575972Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9B8085B3C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:34:38.670552834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:34:38.67504418Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9B8126A992, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:34:40.385004587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:34:40.389263609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9BE7520AFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:34:40.521975801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:34:40.526973944Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9BEF86F2B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:34:42.176818255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:34:42.179524648Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9C52058D2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:34:43.526303559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:43.778857987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:34:43.781280049Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9CB180DD45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:34:44.025780951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:34:44.028301581Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9CC0395095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:34:48.52617458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:53.525760109Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:34:53.525820383Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:34:53.525830809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:34:57.174011581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:34:57.177772253Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEF9FD0003EA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:34:58.526169378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:35:03.526605843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:05.479685139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:35:05.482188471Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA1BEF9951F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:35:05.843388005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:35:05.845871212Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA1D4A7ADA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:35:05.950201515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:35:05.954612416Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA1DB1FAFB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:35:08.208573524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:35:08.21105906Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA261A20D97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:35:08.525925883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:11.246823953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:35:11.249306471Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA316B89A66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:35:11.754525236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:35:11.757036917Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA334FCAF0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:35:13.526112775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:16.078391111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:35:16.080775819Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA436B4049D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:35:17.770333042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:35:17.772797875Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA49B8EA524, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:35:18.526593093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:20.592597564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:35:20.595108136Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA543C73826, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:35:22.296329783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:35:22.298821972Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA5A953A0A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:35:22.460259468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:35:22.464415763Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA5B332FA47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:35:22.464440137Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:22.464465677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:35:22.470219201Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA5B38C0D04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:35:23.508696013Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:35:23.52619613Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:35:23.526230777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:35:23.526227937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:35:28.526225045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:30.598187661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:35:30.602930576Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA7984A0B78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:35:31.329713951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="6.2 MB" avg_comp="3.1 MB" total_uncomp="54 MB" avg_uncomp="27 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:35:31.337429295Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA7C41270B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:35:32.115901237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:35:32.12069752Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA7F2C236C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:35:32.120722301Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:32.120746647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:35:32.122706729Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA7F2E2BBA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:35:33.526655348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:34.579232941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:35:34.58350549Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA8858D9581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:35:38.526757781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:40.750433938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:35:40.755191651Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA9F5688313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T18:35:40.755216628Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:40.755240411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:35:40.757172548Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA9F58ACA72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:35:40.926791875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:35:40.929136006Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFA9FFC88A1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:35:43.124411652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:35:43.126901604Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAA82C629E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:35:43.126920861Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:43.12694054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:35:43.128773973Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAA82E5F2B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:35:43.526562067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:44.868593993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:35:44.870943221Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAAEABB299A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:35:46.968801826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:35:46.971301535Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAB67EB8E23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:35:46.971320134Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:46.971341081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:35:46.973234625Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAB680BF577, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:35:48.526561637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:50.37843436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:35:50.380889381Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAC332702F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:35:50.380907927Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:50.380926593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:35:50.382672384Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAC33447B60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:35:52.380238983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:35:52.382813728Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFACAA788F3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:35:52.382833219Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:52.382855537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:35:52.384686559Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFACAA97FA3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:35:53.526309845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:35:53.526320937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:53.526347414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:35:57.542547623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:35:57.545028471Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFADDE2AF36B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:35:57.545046649Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:35:57.545067096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:35:57.547006684Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFADDE4A8961, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:35:58.526455154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:35:58.539536427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:35:58.542044296Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE199787AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:35:59.714405382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:35:59.716916735Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE5F9E811D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:36:00.106933714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:36:00.109393013Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE77031B10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-30T18:36:00.109409953Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:00.109428821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:36:00.111186854Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE77214A86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:36:00.510460771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:36:00.512962673Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE8F108926, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:36:00.512982927Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:00.51300301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:36:00.514979468Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAE8F329211, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:36:01.781560799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:36:01.784034865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAEDAD4592B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:36:02.317739763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:36:02.320274759Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAEFACADC4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:36:02.320297017Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:02.320319735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:36:02.322386422Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAEFAECDFFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:36:03.181734103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:36:03.184087601Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAF2E47729C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:36:03.525822423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:03.890856622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:36:03.893388962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFAF588D28FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:36:06.886265773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:36:06.890478202Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB00B320214, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:36:08.526714973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:08.784048104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:36:08.790177976Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB07C6DC201, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:36:08.99062217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:36:08.992960354Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB0888535AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:36:08.992978027Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:08.99299653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:36:08.994651195Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB088A1ECFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:36:11.229369608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:36:11.231910821Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB10DF6CF80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:36:11.277215668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:36:11.279694439Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB110D167AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:36:11.279719746Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:11.279743445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:36:11.281531556Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB110F03D0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:36:13.526546296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:14.065549077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:36:14.068021422Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB1B7028F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:36:14.336938462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:36:14.344493154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB1C77EBFE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:36:16.566558039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:36:16.571284614Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB24C37E2EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:36:17.50921911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:36:17.511788726Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB284473967, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:36:17.511809776Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:17.51183402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:36:17.513661616Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB284669511, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:36:18.526580006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:20.550232363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:36:20.552577085Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB33986977E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:36:21.764021557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:36:21.766413599Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB381E0877F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:36:23.50896202Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:36:23.525792572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:36:23.525837007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:36:23.525835492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:24.080920292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:36:24.083210022Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB40BF7A343, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:36:24.083243892Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:24.083269232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:36:24.085270482Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB40C18EF98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:36:24.869691456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:36:24.872030531Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB43AFC091C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:36:27.739616464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:36:27.742019138Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB4E60C58F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:36:28.525835364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:29.691087919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:36:29.693552003Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB55A5E4B24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:36:29.693570363Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:36:29.693590819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:36:29.695441873Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB55A7DA432, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:36:30.306748137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:36:30.30903899Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB57F0E325C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:36:31.747064849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:36:31.749455877Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB5D4E82497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:36:32.899330102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:36:32.90169415Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB61996674B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:36:33.525983807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:37.811816795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:36:37.814219621Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB73E6564A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:36:38.460486792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:36:38.463036034Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB765102C00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:36:38.526535806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:39.228286715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:36:39.230622128Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB792D372BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:36:39.861379733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:36:39.863802962Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB7B8903AE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:36:41.865861987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:36:41.868291559Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB8300A28E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:36:42.59447896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:36:42.596901084Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB85B780B8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:36:43.525979225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:44.278895787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:36:44.281543382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB8BFE16869, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:36:45.473225852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:36:45.47569624Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB9070E6316, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:36:46.044513132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:36:46.046930475Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFB9291BBD84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:36:48.525920523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:53.526282375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:36:53.526323019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:36:53.526379588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:36:56.288473397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:36:56.292319139Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBB8BC643E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:36:58.525819558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:00.241772689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:37:00.244344688Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBC77557D00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:37:03.526268905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:04.036341969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:37:04.04404296Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBD59D170A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:37:04.602744962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:37:04.610935932Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBD7B970357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:37:08.526392375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:09.265661006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:37:09.270456825Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBE915609E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:37:10.513475116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:37:10.516040456Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFBEDB93E75C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:37:13.526118189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:16.461579072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:37:16.463916275Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC03E1925C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:37:17.853010682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:37:17.85551766Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC0910B1935, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:37:18.526829249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:19.796738728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:37:19.799162135Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC104E437C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:37:20.482951887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:37:20.485294054Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC12DCACB8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:37:23.50886763Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:37:23.525760184Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:37:23.525802291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:37:23.525883617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:26.546017866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:37:26.548588531Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC297308907, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:37:28.526738726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:28.550061859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:37:28.552427736Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC30EA0B204, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:37:29.647664486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:37:29.649967278Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC3500C05CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:37:31.621251505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:37:31.623633057Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC3C5B17B20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:37:31.772693503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:37:31.775037559Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC3CEB725AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:37:33.526684048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:34.374390331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:37:34.376715434Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC469C96C2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:37:36.910278597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:37:36.912717012Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC500F1D114, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:37:37.690801344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:37:37.693242902Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC52F768243, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:37:38.526002895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:41.553889752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:37:41.556662112Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC615BEAB1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:37:43.526555581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:37:48.526687424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:37:51.475725861Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDEFC864F7BF09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:37:53.526387701Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:37:53.526435046Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:37:53.526429768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:37:53.535255971Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-06-30T18:37:53.537126711Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010495127s
level=info ts=2026-06-30T18:37:57.666963473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:37:57.674997293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC9D67567CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:37:57.675021133Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:37:57.675048121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:37:57.684235666Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFC9D706D84D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:37:58.526301532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:00.673321468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:38:00.677934143Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCA89750A09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:38:00.788676428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:38:00.791208679Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCA9035E52A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:38:01.677412766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:38:01.681469903Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCAC54665A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:38:02.231917895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:38:02.23653918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCAE65C8371, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:38:03.526787495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:08.258417501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:38:08.266341708Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCC4DC26EAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:38:08.526620589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:08.80627687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:38:08.810928076Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCC6E38696A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:38:13.525904269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:14.073494075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:38:14.075958442Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCDA80A4984, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:38:18.318003139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:38:18.320579539Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCEA50A602A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:38:18.525872755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:20.594886854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:38:20.597327834Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCF2CBFBA66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:38:23.50924697Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:38:23.526520414Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-30T18:38:23.526555302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:23.526550659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:38:23.526581062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:38:23.526700698Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-06-30T18:38:23.526709462Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:38:23.526730354Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.008µs success=true
level=info ts=2026-06-30T18:38:23.545817127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:38:23.54841101Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFCFDCA38357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:38:24.895367954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:38:24.897900609Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD02D14E336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:38:25.995296521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:38:25.997834795Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD06EA470CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:38:27.116200536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:38:27.118677569Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD0B1733ED2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:38:27.321557838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:38:27.323901448Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD0BDAE22F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:38:28.083616475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:38:28.085887716Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD0EB1A2F38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:38:28.526798232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:32.863317732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:38:32.865733909Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD20800ED78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:38:33.526800581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:37.062593309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:38:37.065035809Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD3024CE3BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:38:37.479176883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:38:37.48661217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD31B6DEAEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:38:38.526704393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:38:43.525842041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:48.251038342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:38:48.257018806Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD59D63E55E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:38:48.525966779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:38:52.376117426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:38:52.380112461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD69325C358, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:38:52.400291541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:38:52.404610333Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD69499E567, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:38:53.525769844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:38:53.525817075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:38:53.525812096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:38:58.526594625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:00.099080404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:39:00.101715745Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD85F63545B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:39:03.526382211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:04.229988281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:39:04.232803623Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD9559DCBA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:39:04.644089099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:39:04.64642226Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFD96E461E2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:39:07.37249961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:39:07.375106887Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDA10EA128F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:39:08.525946106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:08.875473482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:39:08.877869784Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDA6A7C4238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:39:10.474206575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:39:10.476676313Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDAC9C81010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:39:12.172291587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:39:12.174738377Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDB2EFF07FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:39:13.526164616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:13.752682664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:39:13.755178764Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDB8D31B261, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:39:18.135837537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:39:18.138312812Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDC9273C2C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:39:18.455135382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:39:18.457564747Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDCA57BE13C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:39:18.526047313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:18.627282774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:39:18.629845562Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDCAFBF5F64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:39:18.708013677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:39:18.710298531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDCB48A9111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:39:19.031262741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:39:19.039752941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDCC82E596D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:39:23.508748569Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:39:23.525962591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:39:23.525969463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:23.52599508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:39:24.55663887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:39:24.562136833Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFDE11574F12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:39:28.52662202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:39:33.526324472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:35.031554413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:39:35.036582793Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE081AB3CD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:39:38.526440822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:41.282394075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:39:41.284547584Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE1F61541A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:39:43.526487779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:44.995418966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:39:44.999819578Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE2D38644DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:39:48.526536532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:50.99271906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:39:50.998575151Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE43913B1F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:39:53.4178788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:39:53.420292406Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE4C96C345B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:39:53.525867044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:53.525881719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:39:53.525912378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:39:56.19675402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:39:56.199059555Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE56F0D5597, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:39:56.524857051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:39:56.527093762Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE5829A9CD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:39:58.526837218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:39:59.119837861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:39:59.122109139Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE61D479E90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:39:59.174099868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:39:59.176367318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE62083B652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:40:01.038213955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:40:01.040536898Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE68FA1499A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:40:01.375471865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:40:01.37773119Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE6A3B97F0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:40:01.700305786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:40:01.709319488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE6B77BD6A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:40:03.526402158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:04.784485362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:40:04.790477306Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE76F230D46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:40:06.625272924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:40:06.627742252Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE7DCA6A71D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:40:08.526503004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:10.805373147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:40:10.807846525Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE8D5CDC888, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:40:11.660267717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:40:11.662529334Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFE908BF3ADE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:40:13.525911347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:18.025416603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:40:18.027932252Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFEA8427BBD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:40:18.526416797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:23.508884549Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:40:23.526315934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:40:23.526365596Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:40:23.526385444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:25.332483413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:40:25.334970434Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFEC37B00CBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:40:28.526237402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:28.60155292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:40:28.604130562Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFECFA8A607D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:40:33.526880118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:36.326516376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:40:36.331126899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFEEC71A0940, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:40:38.526396301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:39.227211006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:40:39.231967024Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFEF74028D19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:40:42.14490663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:40:42.147195121Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF021C61531, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:40:42.424014245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:40:42.426282702Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF032688B29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:40:43.525988071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:44.548566094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:40:44.551108963Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF0B10DE474, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:40:45.437672825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:40:45.440173948Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF0E60BD6F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:40:47.859096578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:40:47.861806893Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF17661E273, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:40:48.526492825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:49.950200979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:40:49.952868535Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF1F306360B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:40:50.748422324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:40:51.496793472Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF24F090789, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:40:53.526508048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:40:53.526551204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:40:53.526569168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:40:55.109277868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:40:55.113694602Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF326A2AE94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:40:55.284947388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:40:55.287391963Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF330FC6335, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:40:55.601620634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:40:55.60400326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF343DC75B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:40:56.89384223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:40:56.896222522Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF390E147CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:40:56.908467478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:40:56.910632637Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF391BE8F62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:40:58.526030638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:41:03.526811371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:06.386484315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:41:06.389552061Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF5C6BAC2F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:41:08.5264075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:09.343689688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:41:09.346332015Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF676F67AC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:41:13.526025545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:13.815570153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:41:13.818185109Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFF7818183AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:41:18.526113617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:23.509135378Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:41:23.526556063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:41:23.526600283Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:41:23.526591355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:24.68120957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:41:24.688153102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFA0968EA5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:41:26.93571493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:41:26.941296728Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFA8FB3BFEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:41:28.526126155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:28.666476619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:41:28.668806425Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFAF6AE6333, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:41:29.749478573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:41:29.751915915Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFB373BF8AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:41:33.526579595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:41:38.526428771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:39.215477866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:41:39.217933699Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFD6B7462C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:41:40.217911534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:41:40.220335969Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFDA732D6C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:41:40.84533977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:41:40.848033942Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFDCC9D24E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:41:43.526620216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:45.602483556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:41:45.604826425Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFEE824FFAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:41:46.533545612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:41:46.535981725Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFF1FA45ED2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:41:48.526777864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:41:49.682875275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:41:49.685368335Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFFDB5C01DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:41:49.835938581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:41:49.843669828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDEFFFE4CA32D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:41:50.845626133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:41:50.851762688Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00020E17201, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:41:51.932375761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:41:51.934794515Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0006170BA36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:41:52.875815818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:41:52.87842395Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00099AD0844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:41:52.90672166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:41:52.909079618Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0009B82B599, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:41:53.52598718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:41:53.526023367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:41:53.52604801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:41:58.526604234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:42:03.526749971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:03.762196505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:42:03.766706805Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00322ADFB6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:42:08.526461322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:12.324299354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:42:12.328302886Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00520FC7C7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:42:12.328324602Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:12.328347638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:42:12.336375772Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0052176ED6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:42:13.501497937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:42:13.505922348Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF005672BDF7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:42:13.505947033Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:13.50597527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:42:13.507794079Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF005674C48F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:42:13.526143006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:13.747565463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:42:13.749856136Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00575B85D00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:42:15.34322803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:42:15.346002734Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF005D4D779B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:42:15.346022048Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:15.346043552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:42:15.347820842Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF005D4F8C8D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:42:15.407788188Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF005D88C302F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:42:18.52668605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:21.694730705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:42:21.697046109Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0074F688753, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:42:21.697070538Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:21.697094216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:42:21.699284342Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0074F894196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:42:23.509148458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:42:23.526542424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:42:23.526587019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:42:23.526663102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:27.294686218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:42:27.297199523Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0089D324D96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:42:27.297224719Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:27.297247523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:42:27.299128438Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0089D51D75F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:42:28.526882355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:29.155490655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:42:29.158028818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0090C1BD9FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:42:29.158048162Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:29.15806903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:42:29.159949881Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0090C3C551F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:42:29.330607118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:42:29.332863032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0091689DA6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:42:29.779698242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:42:29.784132638Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF009316C0998, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:42:30.29437398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:42:30.299806067Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF009502B9C0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:42:30.299831181Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:30.299853435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:42:30.301567599Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0095049F40C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:42:31.901147569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:42:31.903677973Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF009AFC4F6DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:42:33.200803296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:42:33.203103533Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF009FD38D5F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:42:33.203123343Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:33.203146939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:42:33.204898429Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF009FD568C34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:42:33.526368367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:34.425714086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:42:34.428116055Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00A463BCF57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:42:34.428132096Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:34.428152004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:42:34.429869854Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00A4659E05C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:42:36.889944021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:42:36.897403507Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00AD96B297C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:42:37.795009583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:42:37.800228521Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00B0F3A3CCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:42:38.076098217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:42:38.078550304Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00B1FD118E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:42:38.526271179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:41.411186157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:42:41.413624057Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00BE69B8845, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:42:41.413644819Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:41.413666857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:42:41.415411811Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00BE6B8DCD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:42:41.649781468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:42:41.652144943Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00BF4D1689C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T18:42:41.652165712Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:41.713912725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:42:41.716290164Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00BF8A417B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:42:43.455264203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:42:43.457814968Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00C60716C9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:42:43.457833375Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:43.457854689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:42:43.459618185Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00C6090962B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:42:43.526223928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:44.155297611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:42:44.157732042Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00C8A2AC43C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-30T18:42:44.157752169Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:44.157771148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:42:44.159486866Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00C8A48699F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:42:44.932967937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:42:44.935349328Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00CB88378DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:42:44.935391186Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:44.93541356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:42:44.937234457Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00CB8A30E4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:42:44.96740089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:42:44.969520872Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00CBA8FB879, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:42:47.024647046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:42:47.027065532Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00D35311625, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:42:47.63200046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:42:47.634530562Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00D5965C4CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:42:48.015658676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:42:48.017924645Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00D70406367, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:42:48.526755594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:49.905919026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:42:49.908185796Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00DE0EC65D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:42:49.908213969Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:49.908235724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:42:49.910270005Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00DE10DDE07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:42:53.525978797Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:42:53.525984377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:53.526012615Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:42:53.534212862Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-06-30T18:42:53.536061131Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009187104s
level=info ts=2026-06-30T18:42:53.619643079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:42:53.622204333Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00EBE4A0636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:42:55.632786691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:42:55.63512034Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00F364636E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:42:55.903995705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:42:55.906269039Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF00F466E379F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:42:58.526593398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:42:59.588946817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:42:59.591234225Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0102212DD51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:42:59.906313525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:42:59.908719137Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01034FF6671, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:42:59.908738821Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:42:59.9087592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:42:59.910579902Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF010351DD433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:43:00.248804427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:43:00.251112752Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0104967A8F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:43:00.40980538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:43:00.414242435Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01053200972, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:43:00.646932555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:43:00.651878415Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF010614B00AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:43:01.32447678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:43:01.326795417Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0108985C1E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:43:02.781547651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:43:02.783894915Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF010E05F3383, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:43:02.911473234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:43:02.913849645Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF010E81E53AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:43:03.525875362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:05.88022206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:43:05.882657849Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF011991286A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:43:08.526202791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:43:13.525881407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:18.148972716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:43:18.15850311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01474C306B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:43:18.526620814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:18.8013384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:43:18.807446741Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0149B7323ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:43:20.112470164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:43:20.115109244Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF014E9648769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:43:20.817249943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:43:20.819607274Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0151363A877, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:43:23.094029549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:43:23.096466046Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0159B197F4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:43:23.50311519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:43:23.505486133Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF015B3791B2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:43:23.509456028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:43:23.525681842Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:43:23.525801266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:43:23.525808423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:23.525833507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:43:23.525831049Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-06-30T18:43:23.525841223Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:43:23.525858849Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.229µs success=true
level=info ts=2026-06-30T18:43:24.75133841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:43:24.753731338Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF015FDE19ACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:43:26.033143975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:43:26.035630949Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0164A497D2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:43:28.526602407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:29.217937754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:43:29.220544269Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF017081E7B47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:43:30.717310407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:43:30.719787209Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF017617B9B2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:43:33.525898483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:34.206197885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:43:34.208722224Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF018316FFE2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:43:38.526057013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:41.083209413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:43:41.085711087Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF019CB5623DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:43:43.526383195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:43.823940557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:43:43.826340042Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01A6EB07BF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:43:44.083908961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:43:44.086211439Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01A7E2F2C71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:43:45.344187994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:43:45.346604652Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01AC94F7AA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:43:45.897208563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:43:45.899576772Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01AEA4541C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:43:46.074024327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:43:46.078340614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01AF4EBE114, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:43:48.526275231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:43:52.513338822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:43:52.521932002Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01C74FBDC7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:43:53.526323836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:43:53.526383999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:43:53.526413883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:43:58.525948248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:02.316146195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:44:02.31868581Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01EBCEBE25D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:44:03.237161676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:44:03.239637216Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01EF3D1295B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:44:03.525860009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:06.74250332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:44:06.745050825Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF01FC4C10182, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:44:08.526068212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:11.54404064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:44:11.547281746Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF020E2FC4C42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:44:13.52618228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:15.622843828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:44:15.623914766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:44:15.626603525Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF021D6228809, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:44:15.630980154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF021D666F104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:44:15.63099635Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:44:15.631020313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:44:15.64217046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF021D711A50A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:44:18.526563391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:20.008554697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:44:20.013011158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF022DB9727FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:44:23.508662997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T18:44:23.526495588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:23.526513035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:44:23.526546302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:44:23.556808143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:44:23.559288402Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF023AEF5CC29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:44:26.342145553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:44:26.344582742Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02454FB924B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:44:27.067085506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:44:27.071161929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF024804A2441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:44:27.561274809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:44:27.566238857Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0249DCB7E0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:44:27.91840332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:44:27.920759869Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF024B2EE0E99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:44:28.443180336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:44:28.451012504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF024D2887BBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:44:28.526538369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:29.868035253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:44:29.87218974Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF025273D4D19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:44:30.061775478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:44:30.064263739Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02532AFAEB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:44:30.477313015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:44:30.479601244Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0254B736794, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:44:33.526487008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:34.395984295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:44:34.398397435Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02635055E3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:44:38.526632333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:44:43.526392074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:44.329480273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:44:44.331759333Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02885182C3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:44:44.881819666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:44:44.884245985Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF028A6067F21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:44:48.142026215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:44:48.144865239Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF029685E6CB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:44:48.526133909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:49.475973505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:44:49.478426931Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF029B7DC0104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:44:51.652918072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:44:53.005289274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02A8A1383A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:44:53.525793559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:44:53.525828229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:44:53.525867711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:44:56.758676459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:44:56.76346898Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02B6A149E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:44:57.059199088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:44:57.061958985Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02B7BE02ABA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T18:44:58.526304331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:00.693566109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:45:00.696040219Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02C547C4622, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:45:03.526555843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:45:08.52675402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:08.556919354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:45:08.559280893Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E292B1111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:45:09.187209699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:45:09.189658236Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E4EBDC776, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:45:09.915413194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:45:09.920642324Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E7A4F892B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:45:09.989732265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:45:09.995573594Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E7EC79941, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:45:10.053771667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:45:10.05622276Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E826534C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:45:10.373114446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:45:10.375528521Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E956DDB48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:45:10.49375297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:45:10.496418576Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02E9CA0163C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:45:13.526738258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:14.12799242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:45:14.130544963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF02F753E4B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:45:17.140277931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:45:17.142774288Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03028C93502, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:45:18.525830832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:19.15359943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:45:19.156407559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF030A0C82A81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:45:19.231862252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:45:19.234204555Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF030A570E70A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:45:23.509163956Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:45:23.526206788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:45:23.526220649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:23.526234402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:45:25.746725766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="82 kB" avg_uncomp="82 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:45:25.749594222Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03229C714CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:45:26.434648504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:45:26.437191392Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03252C61E89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:45:27.966145203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:45:27.968823374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF032AE112357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:45:28.466302992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:45:28.468985609Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF032CBDF8649, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:45:28.525970884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:29.165794085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:45:29.168270944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF032F58EF2F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:45:33.021636528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:45:33.023842041Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF033DB5FBA27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T18:45:33.526584916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:35.451118071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:45:35.453421545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0346C2FF666, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:45:37.985157295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:45:37.987634366Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF035033D5E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:45:38.110227853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:45:38.112712898Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0350AB176E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:45:38.249946857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:45:38.25719936Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF035134F355A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:45:38.526514478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:43.423567599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:45:43.429774197Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF036479C6DE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:45:43.526384272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:45:48.526433608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:52.43464964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:45:52.438725442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03860971880, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:45:53.525758703Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:45:53.525794614Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:45:53.52582952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:45:56.103216461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:45:56.108943663Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0393B596F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:45:58.525997954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:02.200974264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:46:02.203466026Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03AA69C5BAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:46:03.52683686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:04.256259632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:46:04.258731333Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03B211D8A3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:46:04.372109281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:46:04.37456181Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03B280533C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:46:08.422646749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:46:08.425039073Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03C1971EE03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:46:08.526721235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:08.601006539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:46:08.603339372Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03C24133DCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:46:08.950270934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:46:08.953096944Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03C38E78118, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:46:09.242705946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:46:09.24523056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03C4A54857C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:46:11.147514432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:46:11.150060049Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03CBBDD9CFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:46:13.023174736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:46:13.025760915Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03D2BABA24A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:46:13.526653757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:13.891802974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:46:13.894163198Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03D5F6F29BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:46:16.422099919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:46:16.424580577Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03DF641F1CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:46:16.50311783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:46:16.505200379Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03DFB109BDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:46:17.646659343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:46:17.648986523Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03E3F3D8725, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T18:46:18.526013633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:21.929845992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:46:21.936990251Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03F3ED30834, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:46:23.508947484Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:46:23.52632719Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:46:23.526342276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:23.52636977Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:46:24.279601784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:46:24.285267928Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03FCAC99FEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:46:24.596304643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:46:24.598704573Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF03FDD7A52B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:46:27.923256924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:46:27.925869948Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF040A3C93BC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:46:28.526714063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:29.240176594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:46:29.242613861Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF040F24558A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:46:33.525842573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:46:36.264566138Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF04294D29D06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T18:46:38.525867485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:42.376478832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:46:42.38238142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF044017703BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:46:43.5259372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:46:48.526915783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:49.476676071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:46:49.482780744Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF045A8AE3A72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:46:51.602471783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:46:51.604772502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF046272A3CE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:46:53.514959035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:46:53.517277613Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF046992739CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:46:53.526324212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:46:53.526366845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:46:53.526401881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:55.626158071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:46:55.628534255Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04716FF8FEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:46:56.881546252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:46:56.88386442Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04761D124BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:46:57.957486198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:46:57.959808101Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF047A1F3A76D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T18:46:58.526610371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:46:59.384486329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:46:59.386837676Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF047F7028B57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:46:59.683403008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:46:59.685796639Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04808D4EC62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:47:01.915954268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:47:01.918480465Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0488DE81306, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:47:02.667651974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:47:02.670186447Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF048BAB52145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-30T18:47:03.525903336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:03.968466581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:47:03.970700913Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF049083B388D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:47:06.833673354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:47:06.836089988Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF049B3051676, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:47:08.525858365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:09.657895508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:47:09.660349012Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04A5B5A21BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:47:11.922479732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:47:11.925088422Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04AE258BA64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:47:13.52630046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:14.068546447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:47:14.070724153Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04B623D1EC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:47:15.011825914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:47:15.014233127Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04B9A784744, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:47:16.459316047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:47:16.461573782Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04BF0BE3B53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:47:18.526782108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:18.588894914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:47:18.596142923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04C6FF8714A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:47:23.509132968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T18:47:23.525888593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:23.52589884Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:47:23.525927654Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:47:24.182642122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:47:24.187801604Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04DBD4233E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:47:28.526119257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:33.361482077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:47:33.365883219Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF04FE051C9E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:47:33.52665244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:36.77780921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:47:36.782818811Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF050ABFA1B5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:47:37.247182655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:47:37.249914302Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF050C7D0CECC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:47:38.145784916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:47:38.148343562Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF050FD5F5EB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T18:47:38.526707637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:40.859764552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:47:40.862438998Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0519F244BB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:47:41.272600878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:47:41.275205743Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF051B7BF34A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:47:43.109335294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:47:43.1121874Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF052253C962C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:47:43.526190411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:45.080302334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:47:45.082851494Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0529AB34E50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:47:45.841287653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:47:45.843947952Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF052C811296F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:47:46.701478265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:47:46.704002282Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF052FB546665, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:47:48.526503585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:50.614040147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:47:50.616405307Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF053E48735C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:47:53.526234575Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:47:53.526275728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:47:53.526294445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:47:53.532757316Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-06-30T18:47:53.534564268Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008581404s
level=info ts=2026-06-30T18:47:56.858260721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:47:56.86080485Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05558B823C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:47:58.526001445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:02.226340784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:48:02.228881442Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05698AEC64E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:48:02.704400708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:48:02.706769273Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF056B52C0FB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:48:03.297055294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:48:03.304428109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF056D8CB18AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:48:03.515232233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:48:03.521619204Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF056E5BC5F6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:48:03.52670647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:06.834972943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:48:06.837568646Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF057AB624ED7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:48:08.032590279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:48:08.035135468Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF057F2C37D1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:48:08.427210003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:48:08.429583285Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0580A479396, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:48:08.526058688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:48:13.526374103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:18.473694075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:48:18.476167548Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05A61197F0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:48:18.526628625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:21.536410017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:48:21.538942341Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05B17A70449, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:48:23.509159715Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:48:23.526475459Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-30T18:48:23.526494301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:23.526520565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:48:23.526552991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:48:23.526610223Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-06-30T18:48:23.526617748Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:48:23.526635583Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.318µs success=true
level=info ts=2026-06-30T18:48:25.161080847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:48:25.165308635Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05BEFCB1B91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:48:25.165329728Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:25.165371931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:48:25.171680397Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05BF030261F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:48:28.526239104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:29.219122609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:48:29.223838076Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05CE1B53E8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:48:31.692298366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:48:31.694664129Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05D74FC6FDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:48:31.931294829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:48:31.933686512Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05D833AE313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:48:32.423133526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:48:32.425535855Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05DA08C6030, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:48:33.526523616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:34.437326596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:48:34.439741133Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05E189941C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:48:34.908371804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:48:34.915239817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05E34AC6710, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-30T18:48:35.455589233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:48:35.457979712Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05E554AD6DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:48:36.272628483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:48:36.274940884Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05E85FD5EA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:48:36.835006728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:48:36.83943307Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05EA7A010F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:48:38.526071258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:41.112472786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:48:41.118478087Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05FA6AFFD9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:48:41.1184999Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:41.118523316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:48:41.120327886Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF05FA6CEEDFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:48:43.525935029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:44.11376515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:48:44.116323799Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF060595E6881, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:48:48.525962052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:52.893904356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:48:52.896466011Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06264B5DC57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:48:52.896484642Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:52.896507977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:48:52.8983392Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06264D4A95A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:48:53.526495887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:53.526502631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:48:53.52653334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:48:54.778654915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:48:54.780935915Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF062D5096850, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:48:57.407166065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:48:57.409814177Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06371BAE53F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:48:57.409833112Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:57.4098536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:48:57.411711452Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06371D9AD52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:48:57.751725868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:48:57.754178346Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06386412392, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:48:57.7541955Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:57.7542173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:48:57.755958479Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF063865E65D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:48:58.222398717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:48:58.224800685Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF063A24E83A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:48:58.526810194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:48:58.687193474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:48:58.689786785Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF063BE043C39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:48:58.689806179Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:48:58.689826778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:48:58.691764529Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF063BE243FC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:49:01.622776112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:49:01.631023212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0646D550F9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:49:03.40362866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:49:03.410290556Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF064D7610F28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:49:03.410314093Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:03.410337734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:49:03.412244672Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF064D78238ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:49:03.526218821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:03.976885717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:49:03.979169936Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF064F94A8EEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:49:03.97919083Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:03.979211327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:49:03.981003241Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF064F9688E2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:49:03.992086621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:49:03.995964239Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF064FA4B774B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:49:08.375233835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:49:08.379814241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF065FF96F0C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:49:08.379836542Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:08.379860052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:49:08.381656816Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF065FFB5CBEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:49:08.526402295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:10.651791897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:49:10.654241204Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06687273F55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:49:12.511950825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:49:12.514278547Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF066F6063758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:49:13.065718485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:49:13.068103236Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0671708E952, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:49:13.52601886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:16.142312072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:49:16.144736556Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF067CE6946E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:49:16.144758946Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:16.144780412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:49:16.146643864Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF067CE89D57A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:49:16.794482321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:49:16.796757231Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF067F547C7C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:49:16.854872031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:49:16.857246463Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF067F8E1C8F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:49:17.816112754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:49:17.818642821Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF068323060D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:49:18.526069319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:20.243702707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:49:20.246096955Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF068C2E084E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:49:21.375955859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:49:21.37848321Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF069065E25F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:49:21.378503127Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:21.378525387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:49:21.380348346Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF069067E59D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:49:23.508668211Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:49:23.525772781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:49:23.525806122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:49:23.525840194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:28.024825679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:49:28.027228725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06A92AADC9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:49:28.054281426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:49:28.056567516Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06A946AA0E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:49:28.056590015Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:28.056612655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:49:28.058463933Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06A94897D78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:49:28.526032613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:29.034089729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:49:29.03658252Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06ACED3BBEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:49:29.877015326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:49:29.879405573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06B01115724, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:49:30.412267352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:49:30.414673112Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06B20F82272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:49:31.556931282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:49:31.559329328Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06B6531CF85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:49:31.671012465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:49:31.673393777Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06B6BFDEF0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:49:31.673414637Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:31.673435892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:49:31.675294042Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06B6C1ECB43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:49:33.526328116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:34.705938405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-30T18:49:34.708346084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06C20E3E427, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-30T18:49:34.708380472Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:34.708403283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="107 kB" avg_uncomp="107 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:49:34.710442059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06C21069CB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:49:34.973110237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:49:34.975473559Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06C30D0241E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:49:38.507187472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:49:38.509538653Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06D0376E146, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:49:38.526751853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:42.003152166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:49:42.005651343Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06DD3D844A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:49:43.336150591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:49:43.33867426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06E234D001E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T18:49:43.338697106Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:43.33871886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:49:43.340751378Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06E236D9C8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:49:43.455145036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:49:43.457641828Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06E2A640D3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:49:43.526248133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:44.849399384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:49:44.851935143Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06E7D7EC16A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:49:46.412679769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:49:46.415487388Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06EDAB0C8F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:49:46.415508224Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:49:46.415535056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="19 kB" avg_uncomp="19 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:49:46.417472949Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06EDAD1E7EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:49:46.931478061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:49:46.934013116Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06EF9975DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:49:47.06124411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:49:47.063624955Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF06F0152D38C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:49:48.526371148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:51.644093998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:49:51.646786182Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF070127E989A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:49:53.526111934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:49:53.526150723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:49:53.52614812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:54.248505525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:49:54.251113964Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF070ADBA4D6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:49:54.521179788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:49:54.529229542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF070BE4F64DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:49:55.670283307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:49:55.677157565Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07102B10213, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:49:57.181480353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:49:57.18383814Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0715C893059, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:49:58.526709087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:49:59.490919091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:49:59.493542126Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF071E632620C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:50:01.516183646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:50:01.518657229Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0725EE86FCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:50:02.239311078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:50:02.243550925Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0728A1DE1F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:50:03.000932639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:50:03.006029566Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF072B78F1827, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:50:03.526026968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:06.518543284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:50:06.520880721Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0738911CC2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:50:08.526386587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:09.962004102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:50:09.964592857Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0745653A983, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:50:11.750695549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:50:11.753094196Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF074C0EDB2D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:50:13.526325767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:14.245318052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:50:14.247729537Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF075559FB040, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:50:16.569417313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:50:16.57171125Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF075E024F72A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:50:17.284711976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:50:17.287092453Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0760AC862D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:50:18.526210338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:18.92820165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:50:18.930613743Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0766CBD868E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:50:22.412684844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:50:22.415302541Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0773C723981, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T18:50:23.471166286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF0777B640DB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:50:23.508661032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:50:23.525951774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:50:23.525964291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:23.525983069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:50:25.280976196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:50:25.283501021Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF077E766ACC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:50:28.287626224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:50:28.29022128Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0789A9E5CE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:50:28.526209294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:28.78778727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:50:28.790266807Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF078B86CA0B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:50:33.525906319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:33.537987931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:50:33.540372909Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF079D38E8341, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:50:34.355479173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:50:34.357861637Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07A04477B76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T18:50:38.526767304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:41.555911632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:50:41.558571643Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07BB1790B0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:50:42.110074493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:50:42.112471897Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07BD27D8458, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:50:43.52588892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:47.29775088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:50:47.300435795Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07D07B59AB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:50:47.934392374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:50:47.936765549Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07D2DA5FF49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:50:48.526542558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:49.767261817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:50:49.776544301Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07D9B4DDC5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:50:49.776575022Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:50:49.77661115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:50:49.787741066Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07D9BFAE403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:50:50.562134096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:50:50.566834219Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07DCA68DF54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:50:53.525998591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:53.525998705Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:50:53.526044694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:50:53.81648381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:50:53.818928514Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07E8C3FE490, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-06-30T18:50:53.8189513Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:50:53.818975722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:50:53.820810686Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07E8C5F3F26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:50:54.107468142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:50:54.109800053Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07E9D95D160, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:50:58.526903346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:50:59.171017155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:50:59.175180542Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF07FCB80D30A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:51:00.308544592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:51:00.312827065Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0800F51AF91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:51:03.425585085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:51:03.432791081Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF080C947D683, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:51:03.526529251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:04.522634013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:51:04.527784721Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0810A8AB769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:51:04.957896805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:51:04.960302737Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08124537145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:51:07.317252142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:51:07.319748075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF081B0F54112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:51:08.526452794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:09.46518428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:51:09.467683397Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08230FCFE28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:51:09.871327467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:51:09.873763263Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08249315F7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:51:09.873782077Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:51:09.873801902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:51:09.875574803Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF082494F0D40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:51:11.314670768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:51:11.317102458Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0829F38B52B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:51:13.333073338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:51:13.335582476Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08317881693, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:51:13.52654905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:18.464181394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:51:18.466895854Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08449618FB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:51:18.526287719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:23.508713717Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:51:23.526131616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:51:23.526165972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:51:23.526184272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:25.875662791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:51:25.87810578Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08603201BBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:51:26.175224179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:51:26.17791575Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08614FD89EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:51:26.666941025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:51:26.669273261Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0863247FF20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:51:26.827726193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:51:26.830142534Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0863BDE5381, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:51:28.294986949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:51:28.302610801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08693A33D28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:51:28.526247366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:28.749784878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:51:28.755445866Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF086AEA083A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:51:29.113638593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:51:29.116109228Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF086C41F8606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:51:29.161168304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:51:29.163604087Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF086C6F3FBAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:51:33.526418763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:51:38.526170319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:41.111971839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:51:41.11525049Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0898F54008C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:51:42.438315674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:51:42.441244359Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF089DE5A9FD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:51:43.52681861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:43.763254439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:51:43.767394985Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08A2D684158, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:51:44.500313638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:51:44.506781463Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08A597A9308, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:51:44.649174352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:51:44.651705009Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08A621D3470, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:51:48.526336139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:50.252659772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:51:50.255119612Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08BB01B5A57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T18:51:50.25514159Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:51:50.255163934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:51:50.258981735Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08BB03A2B20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:51:50.280036738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:51:50.282368776Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08BB1BC5942, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:51:52.640567823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:51:52.643336819Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08C3E722FE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:51:53.52622481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:51:53.526260269Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:51:53.526277689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:51:56.016563768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:51:56.019048873Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08D07A960EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:51:58.459282101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:51:58.461778534Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08D9943908A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:51:58.526390741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:03.009749538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:52:03.012257802Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08EA87DFA70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:52:03.526231411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:05.962193541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:52:05.964537907Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08F587713F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:52:06.500802998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:52:06.503573015Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF08F7893634F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:52:08.526614336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:09.445122795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:52:09.447684774Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0902812E39D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:52:10.176033606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:52:10.183343535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09053EC3F9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:52:11.973228941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:52:11.979301154Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF090BEF87640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:52:12.879911788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:52:12.882311275Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF090F4CA9C0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:52:13.526847171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:17.935542684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:52:17.938029239Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0922222A017, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:52:18.52678081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:21.033284708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:52:21.035778345Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF092DAC79CE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:52:21.666295185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:52:21.668862348Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09300837DCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:52:23.50897281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:52:23.526232341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:52:23.526233126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:23.526264971Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:52:24.655403887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:52:24.657985081Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF093B2AD3C36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:52:25.01418922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:52:25.01679867Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF093C810D149, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:52:28.526624588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:29.646637513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:52:29.649179939Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF094DC2DC1BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:52:32.290255084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:52:32.292771066Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09579BFD42A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:52:33.525921258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:33.893761756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:52:33.897859423Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF095D96B78D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:52:35.732052013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:52:35.73800842Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0964718E609, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:52:38.526158503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:39.767008905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:52:39.769474509Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0973764C8D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:52:40.783495458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:52:40.786009345Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09773FB1042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:52:42.429888887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:52:42.432300699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF097D61B8D49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:52:43.52641242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:52:48.52610258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:50.668334854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:52:50.670779189Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF099C129BDA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:52:53.413615473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:52:53.416023936Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09A64CA5DFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:52:53.525780195Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:52:53.525816268Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:52:53.525832078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:52:53.534772791Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-06-30T18:52:53.53666928Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009893566s
level=info ts=2026-06-30T18:52:56.37397085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:52:56.376386903Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09B153DAA66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:52:57.024853403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:52:57.027340442Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09B3C0AC4DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:52:58.526870229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:01.113590496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:53:01.116203663Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09C2FC14F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:53:01.260417468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:53:01.262899131Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09C38808B79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:53:03.526612152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:07.13279583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:53:07.135296478Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09D968500AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:53:07.949777966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:53:07.952286947Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09DC736B3B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T18:53:08.526017819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:08.679331926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:53:08.686564103Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09DF2FC79D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:53:09.822987774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:53:09.829005842Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09E37144C30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:53:12.990718612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:53:12.993221047Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09EF3AE48F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:53:13.526388289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:16.590077396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:53:16.592489753Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF09FCA378A81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T18:53:18.526444946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:23.064571824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:53:23.067160687Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A14C21A2DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:53:23.508826667Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:53:23.525676248Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T18:53:23.525832834Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-06-30T18:53:23.52584439Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:53:23.525843395Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:53:23.525870124Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:53:23.525868133Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.97µs success=true
level=warn ts=2026-06-30T18:53:23.525885931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:25.708916725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:53:25.711427667Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A1E9BE75EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:53:28.526889835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:29.106386065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:53:29.108803925Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A2B43F995A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:53:29.212028652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:53:29.214436725Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A2BA8B6422, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:53:30.748702608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:53:30.751416557Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A316223860, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:53:31.835010237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:53:31.837431964Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A356E2BD3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:53:32.323969331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:53:32.32798104Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A37420DE63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:53:33.525994527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:34.803100166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:53:34.807657601Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A407ED7298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:53:37.649702962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:53:37.65257776Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A4B177E7A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:53:38.526695877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:38.67606264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:53:38.678512316Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A4EEA55AC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:53:43.5267881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:43.786375823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:53:43.78874754Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A61F3E6F80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:53:45.284617564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:53:45.287031526Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A6788B439C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:53:46.418647451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:53:46.421024039Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A6BC22EB31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:53:48.526151493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:49.299652905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:53:49.302291763Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A767DE4978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:53:49.808541765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:53:49.810851395Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A78630A34E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:53:52.397501683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:53:52.399923684Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A820810725, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:53:52.933624835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:53:52.940809176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A840BECFA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:53:53.525770492Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:53:53.525807353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:53:53.525839815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:53:57.375148268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:53:57.381513526Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0A9496DC68B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:53:58.52674275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:54:03.526290649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:54:08.526078549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:54:08.613040308Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF0ABE6E3DB27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:54:09.758413711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:54:09.762287731Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AC2B616838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:54:10.64385889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:54:10.646309041Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AC6012FD14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:54:11.137955891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:54:11.140218777Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AC7D843BAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:54:13.526445461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:15.794738167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:54:15.797173556Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AD93165D77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:54:18.305561509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:54:18.307941501Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AE28BF2213, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T18:54:18.526536249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:19.807325106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:54:19.811423546Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AE825BD55A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:54:19.86840946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:54:19.87445563Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AE861D7AE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:54:21.413862581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:54:21.416159492Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AEE20182CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:54:23.479602917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:54:23.482171177Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AF5D25338A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:54:23.508721281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:54:23.526150126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:54:23.526187545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:54:23.5262251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:24.502773798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:54:24.505197593Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0AF9A1FBBD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:54:26.845868752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:54:26.848425679Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B025CC7057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:54:27.824977457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:54:27.827424654Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B06024EEEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:54:28.526626613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:54:33.526228423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:33.825924809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:54:33.828551385Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B1C5D7D7CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:54:34.257513482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:54:34.259992649Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B1DF8EA1AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:54:36.217690679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:54:36.220256177Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B25465672A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:54:38.526462343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:41.349080973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:54:41.351594948Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B386406327, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:54:43.526367612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:43.841366974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:54:43.843891734Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B41ACCFE93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:54:44.030337387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:54:44.032881723Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B4261081D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:54:48.525925079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:51.663622358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:54:51.66609174Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B5ED0A4782, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:54:52.172488978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:54:52.182303188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B60BCDE8FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:54:53.526499061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:54:53.526524731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:54:53.526554759Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:54:57.407292274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:54:57.413257713Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B743993EDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:54:58.526763904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:01.882327528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:55:01.884818161Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B84E207B49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T18:55:03.526229507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:04.201651543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:55:04.204108023Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B8D85E40B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:55:04.632302512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:55:04.634736639Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B8F208D747, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:55:07.245093977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:55:07.249535576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B98DE421B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:55:07.249559351Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:07.24958444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:55:07.255707088Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B98E42EAB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:55:07.6210411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:55:07.625713021Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B9A44F6D61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:55:08.282538877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:55:08.285054103Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0B9CB9C4661, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:55:08.526170645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:10.588691872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:55:10.591140518Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BA55101B5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:55:11.209166558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:55:11.211651792Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BA7A0CE0AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:55:12.862218205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:55:12.864666574Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BADC93C686, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:55:13.526402569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:16.163853155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:55:16.166391856Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BBA15EAFB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:55:18.526082765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:23.509182302Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:55:23.525919686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:55:23.525944641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:23.525954915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:55:23.870614636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:55:23.875875427Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BD6CE56AF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:55:24.762711542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:55:24.768547739Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BDA219EA3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:55:24.768574057Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:24.768597193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:55:24.770532635Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BDA23ADCDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:55:26.603518367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:55:26.606120256Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BE0FA079F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:55:26.606141795Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:26.60616483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:55:26.60811559Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BE0FC19B85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:55:28.526252402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:31.893749917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:55:31.89629458Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BF4AF2C7D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:55:31.896315523Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:31.896336663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:55:31.89814508Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BF4B11801D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T18:55:33.52646543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:34.189940088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:55:34.192586462Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BFD3D14614, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:55:34.192604303Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:34.192626376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:55:34.194481596Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0BFD3F1509A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:55:35.597102967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:55:35.599672664Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C027B00945, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:55:37.447932115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:55:37.450390476Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C095FF44DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:55:37.450413996Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:37.450435827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:55:37.452339363Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C0961F6598, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:55:37.563887674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:55:37.566321266Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C09CE96538, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:55:37.566338268Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:37.750905283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:55:37.753163621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C0A80CCDB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:55:38.085372546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:55:38.087836502Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C0BBFD7306, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:55:38.087857221Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:38.087879233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:55:38.089728614Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C0BC1E5E88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:55:38.526455266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:41.331000335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:55:41.333593206Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C17D7571C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:55:43.525912347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:43.960926796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:55:43.963323861Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C21A34064A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:55:44.999927628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:55:45.002289149Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C25820CC91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:55:46.309238085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:55:46.313682673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:55:46.315848526Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C2A66C2400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:55:46.31677317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C2A67CDF8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:55:47.776133912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:55:47.778583552Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C2FD9CACF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:55:47.778601413Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:47.778621827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:55:47.780330673Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C2FDB99579, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:55:48.52592521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:49.01347351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:55:49.015839687Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C3475BCED2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:55:50.181263486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:55:50.183507235Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C38CF51003, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:55:50.79628445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:55:50.798621984Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C3B19EBA1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:55:53.52575722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:55:53.525798388Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:55:53.525876565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:53.690374656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:55:53.692634296Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C45E1D8F34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:55:54.813264894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:55:54.815685479Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C4A10D613F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T18:55:55.232235021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:55:55.23466521Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C4BA060EBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:55:58.526734762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:55:58.895222562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:55:58.897801246Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C5945D377D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:55:58.897821022Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:58.897844374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:55:58.899591502Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C5947BFB8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:55:59.786090626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:55:59.788450981Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C5C973BEFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:55:59.986008292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:55:59.988620472Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C5D5604BB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:55:59.988647705Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:55:59.988676675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:55:59.990579363Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C5D5816CDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:56:00.889324463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:56:00.89332275Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C60B4EFD9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:56:00.938618778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:56:00.948924048Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C60E9EA1B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:56:00.948948774Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:56:00.948976507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:56:00.950884228Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C60EC01C7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:56:01.281689235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:56:01.283967585Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C62298242A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:56:01.28399089Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:56:01.284010039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:56:01.285673325Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C622B53F0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:56:03.526592277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:06.095427414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:56:06.097808552Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C74185E2D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T18:56:08.526864937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:08.915201868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:56:08.917677878Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C7E999D9A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:56:11.799475284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:56:11.801891555Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C89582DE22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:56:13.48686612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:56:13.489322847Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C8FA16CD8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:56:13.525880608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:13.666245023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:56:13.668704101Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C904C843E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:56:14.635191296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:56:14.637451301Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C93E86292B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-06-30T18:56:15.553483981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:56:15.556133038Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C975458215, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:56:16.578280738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:56:16.580678179Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C9B259A584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:56:16.802915542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:56:16.805219912Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0C9BFBBBA6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:56:18.526312801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:21.249383077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:56:21.251752246Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CAC8C4ECE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:56:23.508889838Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:56:23.526247757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:56:23.52628382Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:56:23.52634085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:24.22385341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:56:24.22630949Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CB7A1046DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:56:25.428146398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:56:25.430990861Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CBC1DD33D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:56:28.090054191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:56:28.097404092Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CC60CBB85A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T18:56:28.526217557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:28.915408986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:56:28.920231551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CC91D79717, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T18:56:33.526473986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:33.716886286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:56:33.719249813Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CDAFE1D060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:56:34.66922714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:56:34.671650257Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CDE8A76010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:56:34.716939262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:56:34.719314059Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CDEB7E819F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T18:56:34.719336454Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:56:34.719388339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:56:34.721452641Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CDEBA117E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:56:38.526790015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:39.257135426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:56:39.259764309Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CEFA1F1145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:56:41.870366786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:56:41.872662334Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CF95DE2809, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:56:43.142489672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:56:43.146766273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CFE1CED325, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:56:43.526140782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:43.5663113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:56:43.573279978Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CFFB3A8454, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T18:56:43.573307692Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:56:43.573335192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:56:43.575383629Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0CFFB5D15BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T18:56:48.526472519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:48.931743548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:56:48.934457228Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D13AC74361, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:56:48.934479023Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:56:48.934502913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:56:48.93653771Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D13AE9A2CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:56:49.110098035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:56:49.112620304Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D14565D5A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:56:51.705240078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:56:51.707732981Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D1E013EB1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:56:53.525762781Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:56:53.525802896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:56:53.525857888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:56:53.604306176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:56:53.606895034Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D251481CAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:56:54.285223433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:56:54.287739555Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D279DB71C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:56:58.526045735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:00.104263527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:57:00.106797953Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D3D4B4E35D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:57:00.991477656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:57:00.993964494Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D409961D44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:57:02.306803428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:57:02.309281886Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D457FBB734, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:57:03.526406054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:06.175981946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:57:06.178479266Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D53E9A2A0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:57:07.802527122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:57:07.804919585Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D59F8CD193, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:57:08.526258454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:08.609674279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:57:08.612102315Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D5CFA9A2FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:57:11.291173767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:57:11.293694668Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D66F7F0A0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:57:11.465891703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:57:11.468338487Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D679E789BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:57:12.370843583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:57:12.373301156Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D6AFD83822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-06-30T18:57:13.525990241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:15.123488769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:57:15.132058843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D7544725AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:57:15.132083392Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:57:15.132112684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:57:15.14371305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D754FA2A98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:57:15.318411053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:57:15.323139303Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D75FAADE70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:57:15.323167256Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:57:15.432308935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:57:15.434806312Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D766534933, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T18:57:18.526754991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:18.778205411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:57:18.780779709Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D82DC339AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:57:22.95640685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:57:22.958902521Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D926CC770C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:57:23.044097357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:57:23.04648682Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0D92C044C2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:57:23.508751729Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:57:23.526690055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:57:23.526731248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:57:23.526760272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:57:28.526620789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:30.559391757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:57:30.561885087Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DAEBF82B71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:57:30.779549669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:57:30.783838161Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DAF9320F32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:57:31.534732399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:57:31.540057238Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DB2646287B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:57:32.347373106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:57:32.349794939Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DB5688D763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:57:33.300566515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:57:33.30283567Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DB8F593415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T18:57:33.525851104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:35.077965706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:57:35.086278244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DBF9A5790B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:57:35.69218851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:57:35.696537957Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DC1E05AB12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:57:37.149622714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:57:37.152064971Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DC74C67C30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:57:38.526634075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:57:43.525889755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:45.119641472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:57:45.122108157Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DE4FD376EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-30T18:57:48.526656027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T18:57:48.728051969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF0DF26C2D1C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T18:57:48.948634717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:57:48.951388712Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DF34105278, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T18:57:49.49790437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:57:49.50038639Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DF54CAD4B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:57:50.979871069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:57:50.982254625Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DFAD1DF822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:57:51.006398297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:57:51.008536895Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DFAEB0AE75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:57:51.5190616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:57:51.521467163Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0DFCD41F8B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:57:53.526489567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:57:53.526516652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:53.52653475Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:57:53.534178751Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-06-30T18:57:53.536236295Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010208422s
level=info ts=2026-06-30T18:57:54.315260126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:57:54.317699523Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E073ED606D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:57:54.394803777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:57:54.397261375Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E078AADAD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T18:57:58.525857083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:57:59.2025247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:57:59.205077056Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E1973C7D30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:58:03.52676528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:03.587881658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-06-30T18:58:03.590225673Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E29C9C8BA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-06-30T18:58:03.590246362Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:58:03.590268797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T18:58:03.594776152Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E29CE40F1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T18:58:08.526384868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:12.708663488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:58:12.720130333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E4BCC99947, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:58:13.043859507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:58:13.048301788Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E4D05BE042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T18:58:13.526194539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:14.503396366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:58:14.505834077Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E5273C10FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:58:14.508875258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:58:14.510930206Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E5278B8457, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:58:16.477520113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T18:58:16.482248081Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E59D08D645, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T18:58:17.144717784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:58:17.148952991Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E5C4C71448, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T18:58:18.525861235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:20.909736339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:58:20.912292903Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E6A515DD74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:58:20.972306018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:58:20.974646048Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E6A8CF1A7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:58:23.50955877Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:58:23.525818176Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-30T18:58:23.52583496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:23.52583519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:58:23.525866195Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:58:23.525982809Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-06-30T18:58:23.525990988Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T18:58:23.526008995Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.805µs success=true
level=info ts=2026-06-30T18:58:25.479163616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:58:25.481683525Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E7B5722A5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:58:27.633177044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:58:27.637308056Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E835EDBBD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T18:58:28.125966596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:58:28.131540742Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E85363B069, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:58:28.526011058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:30.682970637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:58:30.685373487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E8EB9BDB0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:58:32.519003443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:58:32.521560231Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E9590E8D24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T18:58:32.521581406Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T18:58:32.521604038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:58:32.52365106Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E959307576, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:58:33.324227106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:58:33.326816902Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E9890E7475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T18:58:33.526841833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:34.546152818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="493 kB" avg_uncomp="493 kB" max_age=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T18:58:34.550422448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E9D1FB8253, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T18:58:34.847170532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="168 kB" avg_comp="168 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" max_age=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T18:58:34.852285852Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0E9E3F7BEB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T18:58:35.635784751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:58:35.638542414Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EA12D52BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:58:38.526284646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:41.114908836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:58:41.117445956Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EB5967F7D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:58:41.88907837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:58:41.891463985Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EB878BE5CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T18:58:42.546687615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:58:42.549044207Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EBAEBD8BD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:58:42.990280509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:58:42.992755483Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EBC9304F61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T18:58:43.526537253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:44.614328731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:58:44.616628573Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EC29FB6C4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:58:46.259035883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:58:46.261479694Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EC8C04BAEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:58:47.172337401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T18:58:47.177378966Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0ECC29ADB65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T18:58:48.526565536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:49.671109117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T18:58:49.677887473Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0ED57A50C87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T18:58:53.392557808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T18:58:53.395471775Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EE35395755, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T18:58:53.526406219Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:58:53.526443428Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:58:53.52647523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:58:58.526176682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:58:59.024809368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="612 kB" avg_comp="612 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" max_age=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T18:58:59.028814353Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EF8502FA22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T18:58:59.206652484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:58:59.216177471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EF902C96F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:58:59.325486316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="979 kB" avg_uncomp="979 kB" max_age=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T18:58:59.330368336Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0EF96FB9004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-06-30T18:59:03.525855869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:04.293693681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:59:04.296096677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F0BEF711AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T18:59:04.624342169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:59:04.626704386Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F0D2ABE0FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:59:07.158849407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:59:07.161217451Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F169BE37A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:59:08.482235249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:59:08.484675891Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F1B8A07448, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T18:59:08.526898869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:08.566039344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T18:59:08.568449559Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F1BD9EA91A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T18:59:10.397421236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T18:59:10.40011037Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F22AC9AFBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T18:59:10.677039357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:59:10.679542148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F23B7248FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:59:11.10698398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:59:11.109577942Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F255144AC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:59:12.350513648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T18:59:12.353827828Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F29F3EE00C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T18:59:13.526025637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:15.705728167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:59:15.712040139Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F36768FC0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T18:59:16.776407303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T18:59:16.778779278Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F3A6FE4828, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T18:59:16.934283252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T18:59:16.936998056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F3B06BA298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T18:59:16.99401808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T18:59:16.996418392Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F3B3F714C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T18:59:17.282040747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T18:59:17.286892615Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F3C547C34E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T18:59:18.526019448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:21.105637646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T18:59:21.110690494Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F4A930692A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T18:59:23.509063396Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T18:59:23.526500133Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T18:59:23.526532862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:23.526556088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T18:59:24.814975654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T18:59:24.81742465Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F5862296B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T18:59:25.541557959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T18:59:25.544501404Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F5B175B54B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T18:59:25.971476441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T18:59:25.973959638Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F5CB10A4A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T18:59:27.006331786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T18:59:27.008827127Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F608BE2FAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T18:59:28.52644579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T18:59:33.525880808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:33.911408423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="109 kB" avg_uncomp="109 kB" max_age=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T18:59:33.913972474Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F7A453F572, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T18:59:35.031570337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T18:59:35.034040617Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F7E716AB8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T18:59:35.607890267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T18:59:35.612998797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F809981242, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T18:59:36.043162305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T18:59:36.050145505Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F823A7F916, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T18:59:38.526316195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:39.196398969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T18:59:39.198954494Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F8DF5685AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=info ts=2026-06-30T18:59:43.19640116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T18:59:43.199852453Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F9CDCF1E74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T18:59:43.486614928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T18:59:43.491253199Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0F9DF2D1034, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-06-30T18:59:43.526719134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:46.502188484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T18:59:46.504504561Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FA92C88CB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T18:59:46.599880183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T18:59:46.602208443Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FA989BB632, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T18:59:47.929316682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T18:59:47.931917437Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FAE7DCA524, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T18:59:48.526346411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:49.7476521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T18:59:49.750056195Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FB543C70F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T18:59:53.525921259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T18:59:53.525964251Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T18:59:53.525964678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:54.235821853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T18:59:54.23834564Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FC5FC12D33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T18:59:55.853849587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T18:59:55.856317971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FCC0301D9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T18:59:55.922689246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T18:59:55.929999121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FCC496E64B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T18:59:57.785280886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T18:59:57.791605736Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FD338B123F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-30T18:59:58.17789744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T18:59:58.180327454Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FD4AB6A89A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-30T18:59:58.52659333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T18:59:59.420407069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T18:59:59.42272697Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FD94C52D53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:00:00.610309185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T19:00:00.612832933Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FDDBB3C535, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T19:00:03.179406345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:00:03.181782435Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FE74D42FD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:00:03.525893357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:04.153477416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:00:04.15597835Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FEAEE43E2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:00:07.447019944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:00:07.452208122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FF735A3F08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:00:07.996093516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="171 B" avg_comp="171 B" total_uncomp="24 kB" avg_uncomp="24 kB" max_age=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:00:08.001581025Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FF941BA44D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:00:08.526056447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:08.826797775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:00:08.829273518Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF0FFC57107F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:00:12.60857526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:00:12.611374857Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF100A6DD60D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T19:00:12.947348037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T19:00:12.949969519Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF100BB0DA561, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-30T19:00:13.526420954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:18.370482698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:00:18.37326161Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF101FE4D3E5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:00:18.525848192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:19.892763712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:00:19.895262735Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10259069EDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:00:20.122124448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:00:20.124690779Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10266B44CAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:00:23.508896216Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:00:23.52649435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:00:23.526535429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:00:23.526534982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:23.801463657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:00:23.80388461Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF103420065FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T19:00:25.396699421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:00:25.398998451Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF103A113F8AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:00:26.04715763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:00:26.04957105Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF103C7DA6858, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:00:28.525940358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:32.1529342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:00:32.155441403Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10533CA6126, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T19:00:33.132965241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:00:33.137841009Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1056E579E5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:00:33.526249079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:36.925475882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:00:36.932654959Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1065088DF4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:00:38.149976472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T19:00:38.15467705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF106995F11E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T19:00:38.161703061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="197 B" avg_comp="197 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T19:00:38.164016431Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10699EE6598, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T19:00:38.526373053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:40.374374214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="131 kB" avg_comp="131 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" max_age=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:00:40.379442825Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1071DF962D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:00:41.165976222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:00:41.168422078Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1074D01CE30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:00:43.155484113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:00:43.158014313Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF107C3981113, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:00:43.464260744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T19:00:43.466668699Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF107D5FEBA93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-30T19:00:43.52615204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:45.214705397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T19:00:45.217108859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1083E546160, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T19:00:45.807864059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:00:45.810221958Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10861AE345D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:00:46.247295852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:00:46.249723445Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1087BE132DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:00:47.96447917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:00:47.967247799Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF108E23EDBDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:00:48.526087964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:48.814885305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:00:48.81741622Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10914EB9E83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:00:51.258946827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:00:51.261655962Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF109A69C5067, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:00:52.504403244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:00:52.506985067Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF109F0D63371, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:00:52.94732858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:00:52.949838005Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10A0B3C4FDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:00:53.526717332Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:00:53.526710454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:53.526746639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:00:55.201127325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="9.9 MB" avg_comp="3.3 MB" total_uncomp="86 MB" avg_uncomp="29 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:00:57.085512088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10B01BB2965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:00:58.526060746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:00:58.718491212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T19:00:58.723011808Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10B63573B51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-30T19:01:01.468859933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:01:01.471167535Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10C07246AB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:01:01.530410425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:01:01.53444549Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10C0AEAB26E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:01:01.743114456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:01:01.747259818Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10C1799C787, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:01:03.526600208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:03.945775318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T19:01:03.948237532Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10C9AC9FC19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T19:01:07.876647252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:01:07.879017768Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10D851599D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:01:08.525849348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:09.613338249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:01:09.616042958Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10DEC9D17CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T19:01:09.899464531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:01:09.901688856Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10DFDA6312F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:01:11.546946375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:01:11.549378145Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10E5FD93FBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:01:13.526679495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:13.761885125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:01:13.764267676Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF10EE3DF6536, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T19:01:18.526497261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:18.587606661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:01:18.593619887Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11003B96F48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:01:23.275347017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:01:23.282587219Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1111B349331, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:01:23.508812061Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T19:01:23.526376956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:23.526394584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:01:23.526418272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:01:25.625849435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T19:01:25.630556328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF111A72581D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-30T19:01:25.630580398Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:01:25.630605296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="118 B" avg_comp="118 B" total_uncomp="195 kB" avg_uncomp="195 kB" max_age=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:01:25.632767871Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF111A74BE8F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:01:28.526804586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:29.664668314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:01:29.667576268Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11297C5EE3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:01:30.222223688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T19:01:30.224727859Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF112B8FCA0EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T19:01:30.640087783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:01:30.642759594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF112D1E7F43B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T19:01:32.221740662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:01:32.224276691Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1133029AD36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:01:32.991479348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T19:01:32.993937104Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1135E0C95D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=warn ts=2026-06-30T19:01:33.525943933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:33.70737685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:01:33.709653065Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11388B4C76E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:01:37.401219619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:01:37.403685388Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11464E39B61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:01:37.639388475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T19:01:37.643411391Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF114732A781E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T19:01:38.2191415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:01:38.221645883Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11495A421F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:01:38.526861871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:39.449641288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:01:39.452147195Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF114DEFB2FC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:01:40.197480717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:01:40.200047495Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1150B8F0494, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:01:40.247216914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:01:40.249897468Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1150E876336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:01:40.620155198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="342 B" avg_comp="342 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T19:01:40.622562741Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11524C042CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-30T19:01:40.622581717Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:01:40.622603618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="67 kB" avg_comp="67 kB" total_uncomp="580 kB" avg_uncomp="580 kB" max_age=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:01:40.624601205Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11524E0AB00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:01:42.43185621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:01:42.434285451Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11590BCB8A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:01:42.450326419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:01:42.452609666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11591D48AC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:01:42.64641717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:01:42.648864672Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1159D8631B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:01:43.52592598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:44.412175694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:01:44.41960051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF116070F9A4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:01:48.526560366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:48.648921779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:01:48.655183355Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1170386A369, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:01:53.526399937Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:01:53.52643388Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:01:53.526451865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:54.966246133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:01:54.968651453Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1187BD8EEBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-30T19:01:55.276943514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:01:55.279733941Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1188E616DEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T19:01:55.999034259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:01:56.001510751Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF118B967B19E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:01:56.001531442Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:01:56.001552292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:01:56.003316865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF118B9870A26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:01:57.712943393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:01:57.716946751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1191FA7E0DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:01:57.716966209Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:01:57.716989527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:01:57.72277032Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF119200165CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:01:58.52596462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:01:58.669306678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:01:58.673887706Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11958B0817A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:01:59.470396961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:01:59.475100053Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1198872AAC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:02:00.697892226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T19:02:00.70219495Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF119D19733DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-30T19:02:00.702218667Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:00.702243947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="139 kB" avg_uncomp="139 kB" max_age=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:02:00.704311393Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF119D1B9CA7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:02:02.396404844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:02:02.398997129Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11A36B8E759, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:02:02.399016581Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:02.399039153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:02:02.400843683Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11A36D8D784, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:02:03.525808304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:03.792396443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:02:03.795589085Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11A89F8F10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:02:05.806071105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:02:05.810936944Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11B021742E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:02:06.395493519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:02:06.397966154Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11B251510DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:02:08.526528343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:11.893160114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:02:11.89569223Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11C6CC602D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:02:12.611057452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:02:12.613486385Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11C978DE7A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:02:12.856865937Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF11CA6132229, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T19:02:13.526341538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:17.187848368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:02:17.191958785Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11DA8746563, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:02:17.59216456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:02:17.596561122Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11DC091C1A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:02:17.596581866Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:17.59660609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:02:17.598512206Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11DC0B1F4D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:02:18.526297989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:20.341663212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:02:20.344338238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11E64588A02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T19:02:20.749613954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1da2f4dc67213293 immediate=false num_chunks=1 total_comp="215 B" avg_comp="215 B" total_uncomp="101 B" avg_uncomp="101 B" full=1 labels="{stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T19:02:20.752023396Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11E7CA69A7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-dummy-deploy\"}"
level=error ts=2026-06-30T19:02:20.752044807Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:20.752065286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="76 kB" avg_comp="76 kB" total_uncomp="696 kB" avg_uncomp="696 kB" max_age=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:02:20.755396397Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11E7CDB86AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:02:20.893692981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:02:20.895957592Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11E853B5632, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:02:21.569929764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T19:02:21.572289544Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11EAD8BA7F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-30T19:02:21.850806567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T19:02:21.853140738Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11EBE48582A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T19:02:22.027658417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:02:22.030141486Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11EC8D4E381, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:02:23.508794255Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:02:23.526616791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:02:23.526649818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:02:23.526719335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:25.26860445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:02:25.271135996Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11F8A02332D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:02:26.238913597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:02:26.241402607Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11FC3D63A28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:02:26.241425618Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:26.241448764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:02:26.243292389Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11FC3F73A2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:02:26.630887884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:02:26.633185261Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF11FDB32118A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:02:28.382273435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:02:28.384716134Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12043984EAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:02:28.52629627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:29.187604337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:02:29.189904376Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1207397557F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:02:29.472741142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:02:29.47509977Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12084974164, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:02:30.274483407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:02:30.276954949Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120B460FAD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:02:30.276972847Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:30.276992099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:02:30.278718357Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120B47F94CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:02:30.599204462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:02:30.601549452Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120C7BAF4E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:02:30.601568262Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:30.601589649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:02:30.603462191Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120C7D9A508, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:02:31.105925491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:02:31.108506043Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120E5F19594, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:02:31.444377928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:02:31.4469895Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF120FA1DE1D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:02:33.244667634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:02:33.24714888Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF121656B4DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T19:02:33.52633691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:34.629639597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:02:34.636769874Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF121B83F5DDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:02:38.526045999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:38.916478329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:02:38.922537932Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF122B7B2C035, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:02:38.922562083Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:02:38.922584449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:02:38.924524473Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF122B7D26F25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:02:39.097805096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:02:39.100223847Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF122C2496C7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:02:40.364728509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:02:40.367421491Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1230DD16B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:02:43.52678312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:43.532862061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:02:43.53705917Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF123CABE79C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:02:45.659727127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:02:45.664540141Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF124498C10FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:02:47.367408237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:02:47.370088318Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF124AF344D6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=warn ts=2026-06-30T19:02:48.526364441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:50.787213707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:02:50.789770927Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1257B0AD4EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:02:50.863984717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:02:50.866493943Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1257F9C582F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:02:51.754509067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:02:51.757081407Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF125B4B223AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:02:53.526297674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:02:53.526331307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:02:53.526349547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:53.535512653Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-06-30T19:02:53.537687993Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011535446s
level=info ts=2026-06-30T19:02:54.015032064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:02:54.017689792Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1263B6FB74E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:02:54.182023057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:02:54.184752583Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12645652960, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:02:54.297024305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:02:54.300320118Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1264C493BD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:02:54.934372581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:02:54.94010822Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12672691A08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-30T19:02:58.52670672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:02:58.983552713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:02:58.988368344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12763B5F104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:02:59.178649392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:02:59.183179418Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1276F521E27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:02:59.840201484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:02:59.842686086Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12796A2F5C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:03:01.254062825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:03:01.25665468Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF127EAE83A28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:03:03.526879458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:03.837613508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:03:03.840299392Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12884E7C158, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:03:03.840320128Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:03:03.840342192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:03:03.842176697Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF128850871D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:03:04.721947295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:03:04.72442358Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF128B99BA434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:03:06.206858152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:03:06.212833813Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF129121D74EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:03:06.894160852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:03:06.896627325Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1293B1448DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:03:07.153479798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:03:07.156133154Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1294A8C0339, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:03:08.526586897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:03:13.526542048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:13.545720032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:03:13.548211275Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12AC78CF4AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:03:14.019419606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="19 kB" avg_comp="19 kB" total_uncomp="107 kB" avg_uncomp="107 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T19:03:14.02196351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12AE3C8BB41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-06-30T19:03:14.021982481Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:03:14.022005327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="352 kB" avg_comp="352 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" max_age=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:03:14.029024405Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12AE435F150, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:03:15.884111226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:03:15.889138103Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12B531197F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T19:03:16.160054238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-30T19:03:16.162593195Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12B63607001, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-30T19:03:16.162616119Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:03:16.162639859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="114 kB" avg_comp="114 kB" total_uncomp="782 kB" avg_uncomp="782 kB" max_age=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:03:16.166776905Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12B63A2535F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:03:17.332016518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:03:17.334710398Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12BA93D5741, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:03:18.526761874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:18.585899687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:03:18.588315808Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12BF3F5F454, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:03:19.004897533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:03:19.014164328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12C0D589773, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:03:23.508664847Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:03:23.524960799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:03:23.5256906Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:03:23.525821901Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-06-30T19:03:23.525833001Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:03:23.52585807Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.898µs success=true
level=info ts=2026-06-30T19:03:23.525954394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:03:23.525982888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:03:23.525991581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T19:03:23.529758008Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12D1A7F9303, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:03:23.748851035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:03:23.751079682Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12D27B18749, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:03:23.751098036Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:03:23.75111835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:03:23.752946093Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12D27CEF060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:03:24.231431117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:03:24.240623636Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12D44BD5CE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:03:24.700808701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:03:24.7054385Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12D609136D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:03:27.196272547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:03:27.199046997Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12DF5337049, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:03:28.526463307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:29.283406965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:03:29.286140757Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12E719A13F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:03:33.52614555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:33.693477761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:03:33.696530018Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12F787AA6FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:03:35.405146271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:03:35.411300311Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF12FDEB10D37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:03:38.526173911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:38.618472561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:03:38.620867145Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1309DFF478A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:03:39.560060378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:03:39.562521834Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF130D61FA6F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:03:39.576554633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:03:39.578823899Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF130D719D4C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:03:40.01393181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:03:40.016162851Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF130F12A8ED0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:03:40.937472366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:03:40.940155433Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF131283CD449, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=warn ts=2026-06-30T19:03:43.526709007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:43.579797969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:03:43.582942185Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF131C5C364A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:03:43.789271834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:03:43.796338353Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF131D27AD2B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:03:46.213580026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:03:46.218381248Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13262D730B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:03:48.52661094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:48.701164928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:03:48.703806305Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF132F6F97F5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:03:49.163545056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:03:49.166122826Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF133128AD829, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:03:49.230276276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:03:49.232762892Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF133168463EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:03:49.255015779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:03:49.257440843Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13317FABB6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:03:49.990813458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:03:49.993185736Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13343D75092, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:03:53.180474603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:03:53.183054858Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13401F7C714, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:03:53.526329269Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:03:53.526386666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:03:53.526416222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:03:53.553581422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:03:53.555987744Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF134183393C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-30T19:03:53.556005844Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:03:53.556026485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="41 kB" avg_comp="41 kB" total_uncomp="277 kB" avg_uncomp="277 kB" max_age=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:03:53.557893687Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1341852A053, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:03:53.941728993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:03:53.944118611Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1342F54AEDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:03:56.446853461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:03:56.449509808Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF134C4A9CE4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:03:58.526347855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:00.221106388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:04:00.223573979Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF135A59E1A6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:04:01.39513014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:04:01.397837309Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF135EB9C4B39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:04:02.29654214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:04:02.299592201Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF136215B8453, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:04:03.526303076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:07.085821342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:04:07.092492356Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1373F090896, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:04:08.507164792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:04:08.51546682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13793D969D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:04:08.51549068Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:04:08.515517439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:04:08.52647988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:09.271490229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:04:09.278639239Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF137C1545ABF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:04:09.373930511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:04:09.378704363Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF137C74DCB09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:04:10.359558629Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13801C07397, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:04:10.80675737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:04:10.811225075Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1381CB169C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:04:12.856688859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:04:12.859151701Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13896C2C044, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-06-30T19:04:13.526066214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:17.108783879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:04:17.111538065Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1399438854B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:04:18.526491522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:19.196500984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:04:19.199080344Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13A10A4A9AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:04:19.82239952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:04:19.824756274Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13A35F0E7DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:04:20.904071911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:04:20.912125792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13A76BFE05C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:04:23.50872898Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:04:23.525776946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:04:23.525814172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:04:23.525828935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:28.077415729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:04:28.082978959Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13C222A6087, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:04:28.526688705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:29.155922052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:04:29.158411444Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13C62441F27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:04:29.397036381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:04:29.399558649Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13C70A4594F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:04:29.508953091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:04:29.513651964Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13C7771FFFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:04:29.513674097Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:04:29.513697372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:04:29.520558588Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13C77DC5FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:04:30.363717858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:04:30.368291077Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13CAA61418E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=info ts=2026-06-30T19:04:31.437219173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:04:31.439706068Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13CEA3F0F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:04:31.605103653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:04:31.607688569Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13CF4426406, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:04:33.526513008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:36.940219019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:04:36.942584527Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13E323F4E2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:04:38.52661962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:39.700861791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:04:39.70327635Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13ED6CBA785, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:04:39.703296641Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:04:39.70331689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:04:39.70519237Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13ED6EADA26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:04:40.558608243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:04:40.560959663Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13F09EA38E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:04:41.333248385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:04:41.336008071Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13F381B6E1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:04:42.192310638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:04:42.195549728Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13F6B57FEA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:04:43.526805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:43.748272552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:04:43.754659161Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13FC846308E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:04:44.607753835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:04:44.610056589Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF13FFB4299FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:04:44.784479393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:04:44.786831252Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14005CC3AAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:04:47.401324868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:04:47.40407302Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF140A1C9A8FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:04:47.426235826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:04:47.431281738Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF140A36ACC4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:04:48.28858856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:04:48.298097209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF140D715AE52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:04:48.52602324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:50.274411655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:04:50.279208622Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1414D2A6637, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:04:50.279230224Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:04:50.279251601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:04:50.281050079Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1414D49D7FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:04:52.618758593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:04:52.621332577Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF141D8C3DAAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:04:53.526249956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:04:53.526287826Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:04:53.526286258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:04:53.993410113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:04:53.996047696Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1422AB4904C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-06-30T19:04:54.075178414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:04:54.077784286Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1422F941586, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:04:54.206657194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:04:54.209068237Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF142376813A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:04:55.412709775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:04:55.417242506Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1427F6AA3DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:04:56.141404122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:04:56.14650013Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF142AAE33E1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:04:57.76664565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:04:57.769536465Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1430BA12F60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:04:58.526869052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:05:03.526152564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:04.54486315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:05:04.549631046Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1449FC02EEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:05:07.101479949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:05:07.103911745Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14537FFD82D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:05:08.526700914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:08.69298966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:05:08.695327386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14596D9C81D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:05:13.526107057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:14.018982821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:05:14.021834161Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF146D454D761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:05:14.205295815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:05:14.207747393Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF146DF6AF23B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:05:16.760407114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:05:16.763207775Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14777BB7F57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:05:17.378408177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:05:17.380895676Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1479C8D83DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:05:17.512539712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:05:17.514980987Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF147A48B96B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:05:18.526880312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:21.153562775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:05:21.155895149Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1487D904878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:05:22.057921435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:05:22.063217379Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF148B3A2EEE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:05:22.678816408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:05:22.685103955Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF148D8B5E1A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:05:23.508665475Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:05:23.525999943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:05:23.526029775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:23.526058446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:05:24.387482902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:05:24.389838276Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1493E5157CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:05:24.602775371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:05:24.605139629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1494B27793C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:05:25.683805724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:05:25.686089045Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1498B957CAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:05:27.078390632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=30b8ed815cce331e immediate=false num_chunks=1 total_comp="178 kB" avg_comp="178 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:05:27.081081749Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF149DEB9E70A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-user-workload-monitoring\"}"
level=error ts=2026-06-30T19:05:27.081106591Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:05:27.081140345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="309 kB" avg_comp="309 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" max_age=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:05:27.088401985Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF149DF2B9763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:05:28.526137667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:28.971346955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:05:28.976985104Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14A4FBBA88D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:05:28.977007828Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:05:28.977031296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:05:28.98168653Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14A50051647, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:05:30.034936709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:05:30.039329036Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14A8F0D568C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:05:32.567415994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:05:32.570206414Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14B25E708A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:05:33.526402902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:35.532226514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:05:35.534865199Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14BD69C64A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:05:38.52617921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:38.529322826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:05:38.53174764Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14C893E9113, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:05:40.812197356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="86 kB" avg_comp="86 kB" total_uncomp="979 kB" avg_uncomp="979 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:05:40.814740754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14D1151F370, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-06-30T19:05:40.814760738Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:05:40.81478372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="330 kB" avg_comp="330 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" max_age=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:05:40.821869456Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14D11BE02F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:05:41.188849256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:05:41.195165641Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14D27FD940B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T19:05:43.526271699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:44.103406689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:05:44.10634872Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14DD583AE34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:05:44.539620776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:05:44.544761845Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14DEFA40825, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:05:45.208780591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:05:45.218763694Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14E17D1824C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:05:46.064228354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:05:46.070951953Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14E4A9AE9E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:05:46.498405297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:05:46.503948697Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14E646BD00F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:05:48.010316892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:05:48.015083428Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14EBE7D0592, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:05:48.257721496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:05:48.260120448Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14ECD19DF37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:05:48.526277653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:48.710768462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:05:48.713125676Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF14EE819C6D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:05:53.526557189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:53.526577924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:05:53.526599238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:05:53.556752469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:05:53.559711525Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15008FADA7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:05:56.031161244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:05:56.036302895Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1509C978580, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:05:58.403576271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:05:58.406277267Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15129DB84E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:05:58.526770257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:05:58.798404066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:05:58.800813115Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF151415F8CD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:05:59.788286185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:05:59.790656917Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1517C5F4C0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:06:03.525938784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:06.109312578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:06:06.111784426Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF152F525237B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:06:06.11180326Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:06:06.111824896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:06:06.113647719Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF152F543E67F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:06:06.960419746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:06:06.962813095Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15327DE11CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:06:08.526342203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:10.161950266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:06:10.164691936Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF153E6B52D88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:06:12.382604832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:06:12.387513609Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1546B349A81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:06:13.00153019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:06:13.008062498Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF154903042CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:06:13.526231046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:13.816893197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:06:13.819296473Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF154C08BC6AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:06:14.00274419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:06:14.006029684Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF154CBAC6C0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:06:15.066806446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:06:15.071583947Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1550B2FC37C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:06:15.437680895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:06:15.440419535Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF155212C1554, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:06:15.477472574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:06:15.482553238Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15523AEDEFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:06:16.36816727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:06:16.372887543Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15558BEBFAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:06:18.526615181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:19.194143035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:06:19.196576583Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF156010D304B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:06:21.486212973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:06:21.488713543Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15689AD759E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:06:22.943207687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:06:22.945704042Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF156E0845A75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:06:23.452814029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:06:23.455231603Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF156FEE3C5D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:06:23.508662494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T19:06:23.526368043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:23.52638018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:06:23.526411222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:06:23.843380046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:06:23.845890959Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF157162C8281, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:06:26.206546859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:06:26.209380864Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF157A30C80FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:06:28.526134608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:31.676292512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:06:31.681748258Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF158E93A8ECE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T19:06:33.525884167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:33.535036262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:06:33.537650564Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15957D8910B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:06:33.537671097Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:06:33.537694381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:06:33.539635668Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15957FA8744, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:06:34.127401251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:06:34.129780793Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1597B2581DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:06:35.780322342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:06:35.783034295Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF159DDAE7A51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:06:38.526178843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:43.04549001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:06:43.053159455Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15B8F04C1CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:06:43.209262419Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF15B984A826C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T19:06:43.526868015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:43.8175624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:06:43.823910362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15BBCF5F799, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:06:43.895005892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:06:43.899438015Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15BC175BE2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:06:44.143644491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:06:44.145994779Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15BD02824DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:06:45.764627577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:06:45.766998949Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15C30C6650F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:06:46.291531791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:06:46.293929153Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15C502EE945, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:06:47.207008894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:06:47.209480763Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15C86C146A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:06:48.526704597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:50.138469308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:06:50.140961942Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15D357B4CA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:06:53.526182046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:06:53.526218223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:06:53.526245832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:56.559960903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:06:56.56237224Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15EB43B17B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:06:58.101847868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:06:58.104232923Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15F10212082, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:06:58.526023285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:06:59.218145913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:06:59.220473977Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15F52AB4BD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:06:59.98509774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:06:59.98743534Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15F8061B584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:07:01.892034925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:07:01.894506076Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF15FF20C5DD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:07:03.525935563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:04.557106229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:07:04.559639337Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16090E748B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:07:04.586660563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:07:04.588833143Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16092A62F34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:07:04.752395963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:07:04.754677051Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1609C885908, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:07:07.719990159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:07:07.722771634Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1614D729756, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:07:08.509306156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:07:08.515534504Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1617CB0C9A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:07:08.526607044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:09.785928413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:07:09.788567914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF161C891A986, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:07:10.708769536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:07:10.712225044Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF161FFA008D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:07:10.761256457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:07:10.765788174Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16202D3327A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:07:10.934377768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:07:10.941519479Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1620D4C0029, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:07:13.444043171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:07:13.448572828Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF162A2BA774E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:07:13.526425522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:16.844061441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:07:16.846559524Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1636D439B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:07:18.526099417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:21.717478392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:07:21.720508803Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1648FC5125F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:07:23.508672542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:07:23.526484358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:07:23.526491498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:23.526515777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:07:27.074042414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:07:27.079666723Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF165CF339E52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:07:27.14000174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:07:27.142520451Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF165D2F24F2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:07:28.134702148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:07:28.137252355Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1660E3BC8EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:07:28.153307774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:07:28.155948523Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1660F59A43D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:07:28.155967805Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:07:28.155988404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:07:28.158373745Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1660F80E497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:07:28.526020279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:29.216298572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:07:29.21886563Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1664EB5593B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:07:30.46585403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:07:30.468428064Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF166992E2520, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T19:07:33.52666306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:07:38.526323815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:38.728852125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:07:38.733317483Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16885CF8817, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:07:38.823472914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:07:38.832405311Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1688BBA261F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:07:39.536873163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:07:39.541155821Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF168B5F7113C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:07:40.594402616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:07:40.596722386Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF168F4E1B645, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:07:40.636858175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:07:40.639317283Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF168F76B8583, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:07:40.923479578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:07:40.926102938Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1690883F2BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:07:40.926120702Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:07:40.926143654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:07:40.927973169Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16908A2B274, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:07:42.300859455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:07:42.303506121Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1695A9C7E1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:07:43.070685437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:07:43.073133545Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF169887D1A33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:07:43.526254575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:43.75770531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:07:43.760210483Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF169B16F4176, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:07:46.928147988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:07:46.930550156Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16A6E680A83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:07:48.52630245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:50.692127795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:07:50.694730366Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16B4EC4E173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:07:51.942730523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:07:51.945031392Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16B994CA936, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:07:52.889710925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:07:52.891921618Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16BD1BBF290, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:07:53.52645353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:07:53.52649005Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:07:53.526488388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:53.534583754Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-06-30T19:07:53.536570125Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010550185s
level=info ts=2026-06-30T19:07:53.729162232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:07:53.73156311Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16C03C8C22F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:07:55.809309945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:07:55.811895382Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16C7FC61C56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:07:58.23332769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:07:58.236139783Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16D1044884E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:07:58.526198804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:07:58.966525408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:07:58.969631167Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16D3BFD74C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:08:03.526027978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:06.499548371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:08:06.50568647Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16EFD2CF70E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:08:06.777426417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:08:06.782059996Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16F0DA84204, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:08:07.975602987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:08:07.982397822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16F5532491E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:08:08.201147409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:08:08.20573545Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16F6281E779, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:08:08.353090991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:08:08.356187906Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16F6B7A6DD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:08:08.526746388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:08.855567405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:08:08.859810682Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF16F897F27F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:08:12.260841912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:08:12.263480064Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF170545E4A59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:08:13.526490496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:15.593149737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:08:15.595758762Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1711AFBBFAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:08:18.526559707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:18.752025472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:08:18.754719934Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF171D74571A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:08:18.754738279Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:18.75475819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:08:18.756565138Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF171D765DB17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:08:19.059656445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:08:19.062047109Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF171E9976985, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:08:19.195473867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:08:19.19967408Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF171F1CAB630, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:08:19.19970125Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:19.199732213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:08:19.207652596Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF171F245D052, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:08:19.963476531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:08:19.967984292Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1721F970F18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:08:21.756746727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:08:21.759416615Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1728A5C30A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:08:23.414068771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:08:23.416755301Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF172ED2719EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:08:23.416782893Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:23.416814318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:08:23.419040903Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF172ED4BC1EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:08:23.508673323Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:08:23.526132988Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:08:23.526177563Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:08:23.526195115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:23.526226192Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:08:23.52632213Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-06-30T19:08:23.526329685Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:08:23.526348104Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.973µs success=true
level=info ts=2026-06-30T19:08:25.210893205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:08:25.218486281Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF173588ABB93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:08:27.382841422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:08:27.387551601Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF173D9D4C6F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:08:27.387572369Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:27.38759566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:08:27.389429134Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF173D9F42EC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:08:28.526372443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:29.262112609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:08:29.264673305Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17449B5900A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:08:30.990260221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:08:30.992805482Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF174B0B91D40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:08:32.605227786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:08:32.607868463Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17510FCEFEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:08:32.607886859Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:32.607907651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:08:32.609815599Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF175111CD341, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:08:33.025192168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:08:33.02751845Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1752A008478, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:08:33.5264129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:34.142128186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:08:34.144592416Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1756C958397, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:08:34.144611632Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:34.144634539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:08:34.146486564Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1756CB41CD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:08:36.556142322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:08:36.56037278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF175FC93E343, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:08:37.259921926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:08:37.264705081Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF176268E79D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:08:37.405637635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:08:37.408046215Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1762F1907E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:08:37.408065122Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:37.408084675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:08:37.410174981Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1762F3BE5FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:08:37.480271184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:08:37.482812084Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF176338EC78F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:08:38.175467918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:08:38.178018858Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1765CFE616B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:08:38.369546803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:08:38.37183704Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF176688C8599, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:08:38.526652387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:40.928675598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:08:40.931289635Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1770119C0DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:08:42.762716947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:08:42.765183673Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1776E68636C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:08:43.525906221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:44.120635419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:08:44.123230095Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF177BF5AD0E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:08:45.46518429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:08:45.467530826Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1780F7C39F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:08:45.467549798Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:08:45.467569111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:08:45.469301915Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1780F99491A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:08:48.525860387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:49.496622826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:08:49.501504686Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF178FFEAE878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:08:50.57663276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:08:50.583177456Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17940657DE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:08:50.907222491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:08:50.909663534Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17953DBD89F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:08:52.322483295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:08:52.325445719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF179A83D1FF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:08:53.525978001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:08:53.525974677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:08:53.52601677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:08:53.958744481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:08:53.963814887Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17A09E864DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:08:57.536861961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:08:57.542117972Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17ADF2B91CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:08:58.423698427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:08:58.427770891Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17B13FA4295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:08:58.526283867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:00.572577862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:09:00.574887218Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17B93F3FE01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:09:01.769714843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:09:01.772241332Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17BDB5092E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:09:01.772264409Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:01.772286483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:09:01.774279198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17BDB73C7A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:09:03.525960181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:03.596103802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:09:03.598346172Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17C482B34BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:09:05.749410579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:09:05.751872383Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17CC8856C26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:09:05.751892675Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:05.751915102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:09:05.753756188Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17CC8A5946B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:09:06.279704047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:09:06.282401896Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17CE825B7A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:09:06.717889081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:09:06.722166837Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17D025B0475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:09:06.937485882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:09:06.940102377Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17D0F58AB21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:09:08.526479225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:09.023876103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:09:09.028428925Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17D8BD01E22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:09:09.206913179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:09:09.211386412Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17D96BA26AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:09:10.6681299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:09:10.670586017Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17DEDB3F80D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:09:10.861197843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:09:10.863699339Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17DF936B47F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:09:11.571966837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:09:11.574387702Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17E23922B80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:09:13.346972647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:09:13.349319462Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17E8D5E4FE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:09:13.349342502Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:13.349377431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:09:13.351262008Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17E8D7E782B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:09:13.525814253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:16.69481349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:09:16.69717411Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17F54EAC8FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:09:16.90110556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:09:16.908442984Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17F61828C2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:09:16.997493662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:09:17.001724967Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17F6711227A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:09:18.003637016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:09:18.006120591Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF17FA2EE900E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:09:18.526119539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:20.955555146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:09:20.957916003Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18052E0064B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:09:21.492781867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:09:21.495055242Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18072E458EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:09:21.495075027Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:21.495096189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:09:21.496839476Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18073021A8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:09:21.57395194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:09:21.576450907Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18077BC824E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:09:23.509379222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:09:23.526752174Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:09:23.526786769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:23.526796839Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:09:25.531108587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:09:25.533699026Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF181639CAC67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:09:28.049194505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:09:28.051739278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF181F9B2BC87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:09:28.525944607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:29.558265918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:09:29.560901061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18253A61739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:09:29.690646953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:09:29.693137092Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1825B88C8E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:09:30.13151287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:09:30.134136454Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18275D0C715, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:09:32.051149967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:09:32.054135466Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF182E8415779, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:09:32.054156507Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:32.054180709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:09:32.056345001Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF182E8663AC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:09:33.526456454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:38.351957718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:09:38.354527314Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1845FCA7378, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:09:38.526328622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:40.77820384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:09:40.78338007Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF184F09065CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:09:42.677305373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:09:42.684812252Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18561E5A581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:09:43.526139556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:44.792130921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:09:44.797200554Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF185DFCEF37E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:09:45.526617546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:09:45.531114867Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1860B8CA2D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:09:48.5263766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:52.181717782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:09:52.184240605Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF187981ABB11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:09:52.187269372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:09:52.189947283Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1879874D7D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:09:52.189962354Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:09:52.189984461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:09:52.194338697Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18798B82C2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:09:52.838656504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:09:52.843420262Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF187BF64DE22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:09:53.526630071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:09:53.5266648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:09:53.526692625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:54.446219567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:09:54.448605038Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1881F1438DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:09:55.047598883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:09:55.050069515Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18842EBC0C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:09:55.10231517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:09:55.104637768Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF188462E6E56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:09:55.509409096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:09:55.512044424Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1885E757BEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:09:55.692374943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:09:55.694775988Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF188695A5AF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:09:55.967731245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:09:55.97009911Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18879C30AD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:09:58.526978893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:09:58.750412095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:09:58.752895537Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1891FA13E1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:10:00.966019712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:10:00.969002422Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF189A3B6C040, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:10:03.526123833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:03.63040256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:10:03.632853955Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18A42801AE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:10:04.201883409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:10:04.209248665Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18A64DAB275, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:10:04.971477564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:10:04.979239591Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18A92BE9999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:10:05.064434107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:10:05.068788578Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18A98160F96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:10:05.6715834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:10:05.673986262Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18ABC28C3E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:10:05.67400896Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:10:05.674032132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:10:05.676006788Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18ABC490857, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:10:05.866600936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:10:05.869410985Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18AC7CDBF88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:10:06.169239937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:10:06.174080988Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18AD9F55194, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:10:07.930247235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:10:07.932605944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18B42C93C23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:10:07.93262806Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:10:07.932650279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:10:07.934480198Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18B42E792C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T19:10:08.526086006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:09.696997671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:10:09.699887784Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18BAC2028E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:10:13.453517725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:10:13.458480591Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18C8C27109C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:10:13.525970798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:15.278316396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:10:15.280565501Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18CF8C327DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:10:15.659919393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:10:15.662410408Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18D0F84D09A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:10:18.526133049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:19.504242456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:10:19.506774076Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18DF4A921DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:10:23.508724169Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:10:23.526117565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:10:23.526150301Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:10:23.526145881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:23.543540298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:10:23.545886031Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18EE56A16C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:10:24.401643342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:10:24.404618199Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18F188F90C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:10:25.978919959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:10:25.982130854Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF18F769F86C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:10:28.518135075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:10:28.524062234Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1900E1F1FF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:10:28.526096223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:10:33.526413959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:34.728416845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:10:34.73420699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19180482E32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:10:35.654050094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:10:35.658657407Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF191B76241D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:10:36.37648312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:10:36.379390263Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF191E2583E22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:10:37.721525728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:10:37.726070255Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF192329C881F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:10:38.525824507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:39.641087651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:10:39.643558194Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF192A4E7C459, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:10:41.771919641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:10:41.774439769Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19323EAD0BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:10:43.188046768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:10:43.190343452Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF193784F3957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:10:43.526847953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:43.535941854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:10:43.538181376Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1938D0B870E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:10:45.544185389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:10:45.546596438Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19404C09B07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:10:46.465977985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:10:46.468185881Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1943BB0283E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:10:47.535693448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:10:47.538563339Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1947B7C4A6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:10:47.976452764Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF19495983F4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T19:10:48.526405462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:49.51986011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:10:49.529049852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF194F2201111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:10:49.529072986Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:10:49.529103059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:10:49.537973789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF194F2A73739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:10:50.857291192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:10:50.862183928Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF195419607F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:10:50.925470673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:10:50.929546266Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF195459B57E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:10:51.084803664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:10:51.08707196Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1954EFEA67A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:10:53.525918873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:10:53.525959986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:10:53.525980119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:56.244113363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:10:56.246863346Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19682885543, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:10:57.367661243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:10:57.370266211Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF196C57EFB0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:10:58.526133384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:10:59.909277281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:10:59.91198668Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1975CFEE959, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:11:00.520943512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:11:00.523596276Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1978172D318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:11:02.494105174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:11:02.496653321Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF197F70E58F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:11:02.496672089Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:11:02.496693543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:11:02.49854053Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF197F72D9832, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:11:03.015743386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:11:03.018047775Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19816228085, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:11:03.019020504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:11:03.020852734Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF198164F5F1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:11:03.224326574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:11:03.226807433Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19822936BF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:11:03.297883769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:11:03.302210518Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19827129CA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:11:03.52617513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:06.292294956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="14 MB" avg_comp="3.4 MB" total_uncomp="119 MB" avg_uncomp="30 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:11:06.303527321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF198D9F6C05F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:11:08.526137649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:09.937641864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:11:09.942520425Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF199B2D9F07F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:11:13.526494788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:14.96494386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:11:14.968594819Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19ADE710D4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:11:17.187485604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:11:17.192693728Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19B630102E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:11:18.065222031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:11:18.069850595Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19B974A1F2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:11:18.526624239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:21.045919829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:11:21.048438004Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19C48D40F01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:11:22.761236038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:11:22.763831809Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19CAF123774, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:11:23.312723559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:11:23.315098582Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19CCFEDCB52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:11:23.315119378Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:11:23.315140972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:11:23.316922827Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19CD00CC65E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:11:23.508683807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:11:23.526708786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:11:23.526746632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:11:23.526748615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:26.73881431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:11:26.741320062Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19D9C2582B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:11:27.991867842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:11:27.997585203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19DE70609FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:11:27.997613674Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:11:27.997639723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:11:28.006063061Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19DE7897915, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:11:28.113404995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:11:28.11879944Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19DEE40BBD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:11:28.52614533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:29.914124993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:11:29.918542404Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19E598585FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:11:29.954498957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:11:29.956884545Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19E5BD0EC97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:11:32.221774124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:11:32.224260449Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19EE2F4CADF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:11:32.84914034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:11:32.851638702Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19F085ACF69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:11:33.401578729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:11:33.404135217Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF19F29489C01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:11:33.526639122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:11:38.526274165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:39.85295304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:11:39.860281407Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A0AA193D4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:11:39.986689295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:11:39.991286628Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A0B1E8AA46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:11:40.227047456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:11:40.229544595Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A0C01CFCA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:11:41.086283679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:11:41.088719817Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A0F351E949, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:11:42.435541569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:11:42.43854392Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A143C5CABD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:11:43.435835676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:11:43.440427448Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A17F7E6B00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:11:43.526026233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:44.372187708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:11:44.376890681Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A1B74FF751, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:11:45.990892465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:11:45.995466248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A217C7AE78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:11:46.537335524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:11:46.539764179Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A2383AAF49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:11:47.282524892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:11:47.285011177Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A264A552D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:11:48.526711967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:50.900754622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:11:50.903146285Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A33C4FEF52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:11:51.76578217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:11:51.768508869Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A36FE1E10A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:11:52.924754429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:11:52.927208048Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A3B4F33CC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:11:53.526254044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:11:53.526299756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:11:53.526309252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:11:58.525883874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:11:58.851089368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:11:58.853485715Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A5162E2187, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:11:59.884338871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:11:59.888564666Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A553E1408D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:12:02.927480211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:12:02.934466744Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A6096BD27A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:12:03.526626706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:04.279315295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:12:04.281813893Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A659BC2D17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:12:04.616078623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:12:04.619402639Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A66DDCE6F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:12:04.619427411Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:12:04.619451258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:12:04.62451924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A66E2BBCFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:12:08.526201359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:12:13.526408481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:16.340957048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:12:16.346339962Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A928D6665A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:12:16.818500589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:12:16.821467645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A9452955DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:12:16.821489835Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:12:16.821516739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:12:16.826179819Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A945702B46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:12:17.588170991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:12:17.592537065Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A9731D28B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:12:18.300703197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:12:18.303120333Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A99D786639, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:12:18.526730178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:18.989519446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:12:18.99184968Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A9C6866C98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:12:19.389231767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:12:19.391732786Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A9DE5B788B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:12:19.951931266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:12:19.955209271Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1A9FFF188CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:12:21.292156762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:12:21.299402684Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AA50100ABA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:12:23.508659798Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:12:23.526390286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:12:23.526410763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:23.526425297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:12:25.726408248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:12:25.730890818Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AB5833B557, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:12:26.208197324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:12:26.21104043Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AB74D2AA97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:12:28.526260404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:31.899322088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:12:31.904265283Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1ACC8285CC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:12:31.95227759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:12:31.955054836Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1ACCB3115B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:12:31.987079179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:12:31.99150981Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1ACCD5CFB11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:12:33.526120061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:33.881136826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:12:33.888512646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AD3E6F3DBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:12:34.035968949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:12:34.040302157Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AD477AD4BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:12:37.403644528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:12:37.406082441Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE10193044, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:12:38.136942265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:12:38.139626846Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE3BD34671, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-30T19:12:38.139651583Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:12:38.139676769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:12:38.144238922Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE3C1A4B0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:12:38.526169501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:38.598403284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:12:38.602457919Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE5768E2F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:12:38.660481064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:12:38.663167896Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE5B0540AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:12:39.008326563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:12:39.01074963Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AE6FBDFC47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:12:40.576662455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:12:40.580875845Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AECD54ED74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:12:40.685372017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:12:40.689877829Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1AED3D3B29A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:12:43.526827579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:46.272047817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:12:46.274428344Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B020B1E978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:12:46.380796975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:12:46.383275625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B0272D14BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:12:46.898893695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:12:46.901282914Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B0460EB1F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:12:46.901302519Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:12:46.901324679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:12:46.903261519Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B0462E9136, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:12:48.526552259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:50.547549593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:12:50.550019505Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B11F8973DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:12:51.071509986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:12:51.073995719Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B13EC52B73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:12:51.158370015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:12:51.160684974Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B143F082DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:12:53.52637718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:12:53.526415505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:12:53.526420025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:53.534603524Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-06-30T19:12:53.536892554Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010391552s
level=info ts=2026-06-30T19:12:54.821918812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:12:54.824757005Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B21E555D0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:12:57.846037534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:12:57.852920017Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B2D2D3281C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:12:58.526963885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:12:58.877292144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:12:58.882456715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B3103016F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:13:02.064978187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:13:02.067485619Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B3CE07D750, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:13:02.829319695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:13:02.831938954Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B3FB9879A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:13:02.993416565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:13:02.998724128Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B4058A5CD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:13:03.526551323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:03.598789155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:13:03.603782008Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B42999F03C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:13:06.273566065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:13:06.278645091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B4C909D229, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:13:06.452227863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:13:06.456840962Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B4D3A7E214, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:13:08.526275331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:10.069162636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:13:10.071613686Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B5AB1E72CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:13:13.526487439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:15.251319508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:13:15.253944672Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B6E00176BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:13:16.659839275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:13:16.662350302Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B733F33831, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:13:16.754325476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:13:16.756794141Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B73994B57D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:13:18.526403955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:21.98743151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:13:22.033322864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:13:22.039290786Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B87470BDA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:13:22.305769921Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B884530270, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:13:23.508678783Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:13:23.526616626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:13:23.52661283Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:13:23.526644745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:13:23.526717026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:23.526760595Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-06-30T19:13:23.526768578Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:13:23.526787277Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.763µs success=true
level=info ts=2026-06-30T19:13:23.775481249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:13:23.781642334Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B8DC4B0266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:13:25.718776372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:13:25.72340593Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B950081677, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:13:26.232871629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:13:26.235370988Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B96E8CB087, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:13:27.150934102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:13:27.153579076Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B9A546CE7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:13:27.942866147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:13:27.945419874Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B9D4793313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:13:28.52689835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:28.569077229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:13:28.571749802Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1B9F9CECB66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:13:29.468071244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:13:29.470744226Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BA2F64BED4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:13:30.632633316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:13:30.635524982Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BA74D21A29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:13:31.047681753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:13:31.052018916Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BA8DA524D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:13:31.05204313Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:13:31.052068441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:13:31.053922588Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BA8DC47886, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:13:33.100916441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:13:33.103225792Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BB07E82400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:13:33.526816287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:34.265911304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:13:34.268512017Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BB4D5CDBF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:13:35.081980974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:13:35.084395974Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BB7DFDFA1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:13:37.565670688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:13:37.568457887Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BC120CA778, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:13:38.526703323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:39.102046229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:13:39.105277818Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BC6DA79699, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:13:40.249933272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:13:40.255124591Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BCB230D2A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:13:42.153477589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:13:42.155899724Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BD237D050D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:13:43.525953015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:44.374767149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:13:44.377533544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BDA7E8EAB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:13:48.526482848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:48.571660175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:13:48.576286088Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1BEA22B4BBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:13:53.526164416Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:13:53.526206094Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:13:53.526217872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:56.852588046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:13:56.857309863Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C08FC33FDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:13:57.444713713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:13:57.449203233Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C0B30A0AA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:13:58.077181676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:13:58.079555166Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C0D89D1B60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:13:58.525963024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:13:59.361558728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:13:59.363910427Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C1252AC061, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:13:59.893367639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:13:59.896209359Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C144E544D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:14:03.526190092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:04.524423491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:14:04.529547593Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C2590F7364, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:14:05.93801834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:14:05.946409334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C2AD80EFE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:14:07.776754396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:14:07.781390156Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C31AE2A4CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:14:08.52626794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:08.680604895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:14:08.68309312Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C350A1DF4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:14:09.124659614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:14:09.127182406Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C36B1899CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:14:09.174190347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:14:09.176611059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C36E0CCB63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T19:14:13.526703673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:13.967472767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:14:13.970403951Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C48BC6FC5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:14:13.980403124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:14:13.98503413Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C48CA6496D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:14:14.310505527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:14:14.31463219Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C4A04C2D17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:14:17.519640743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:14:17.525775815Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C55FB14B6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:14:18.526577666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:19.224181608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:14:19.226557372Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C5C511A8AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:14:19.879178962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:14:19.882034994Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C5EC217B82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:14:20.076609894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:14:20.079250221Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C5F7E47294, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:14:21.079241346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:14:21.081981415Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C633A71DED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:14:22.504974259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:14:22.507607872Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C688A1CFFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:14:22.507632067Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:22.507655865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:14:22.509722195Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C688C53078, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:14:23.445718959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:14:23.448277485Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C6C0B35D64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:14:23.448302485Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:23.448326479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:14:23.450537538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C6C0D80BD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:14:23.508910815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:14:23.526698656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:14:23.526744237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:14:23.526775497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:26.908614665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:14:26.911871067Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C78F268134, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:14:27.717324688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:14:27.722279708Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C7BF74D451, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:14:27.722302171Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:27.722326622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:14:27.724261216Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C7BF9431D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:14:28.525901662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:28.860227648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:14:28.862608686Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C8036B26F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:14:29.325951874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:14:29.328254177Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C81F2C921B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:14:32.631890629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:14:32.634462874Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C8E43D5F5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:14:33.526415168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:35.604163172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:14:35.606871769Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1C99569A334, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:14:36.334782882Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF1C9C0CFF78B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T19:14:38.526218047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:38.783800784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:14:38.786443779Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CA52EC5BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:14:41.202193974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:14:41.205232654Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CAE3194468, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:14:41.466929854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:14:41.471442471Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CAF2F8997D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:14:42.332004353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:14:42.334438487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CB2668B9A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:14:43.526427891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:48.392541668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:14:48.39517058Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CC8FA71552, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:14:48.395191199Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:48.395214064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:14:48.397227278Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CC8FC9B99A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:14:48.526712154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:50.589941231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:14:50.592385344Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CD129F5F57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:14:51.044746284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:14:51.04720211Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CD2DBAEE7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:14:51.047224603Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:51.047245758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:14:51.049266528Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CD2DDCB3ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:14:51.995680773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:14:51.998048426Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CD6667AB8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:14:53.526749808Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:14:53.526784873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:14:53.526802414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:53.935796431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:14:53.94146582Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CDDA39CCFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:14:54.271487524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:14:54.278844147Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CDEE59ABC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:14:54.571720707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:14:54.577654031Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CE00296C74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:14:56.04968852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:14:56.052340291Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CE580E8FB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:14:56.052389134Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:56.052412011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:14:56.054406333Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CE582FFACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:14:56.535230778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:14:56.539577119Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CE75193005, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:14:56.539600641Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:56.539626135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:14:56.545711306Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CE757810AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:14:57.878825912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:14:57.883383682Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CEC5308AF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:14:57.883420746Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:14:57.883455687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:14:57.885460191Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CEC554FB70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:14:58.311411944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:14:58.526045185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:14:59.646745582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:14:59.65150288Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CF2E955703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:15:00.143480157Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CF4BE77CC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:15:00.906410635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:15:00.911058164Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1CF79A7F4D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:15:03.52676233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:04.781314972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:15:04.783946582Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D0608048BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:15:05.504101228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:15:05.506517715Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D08B921C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:15:06.05908439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:15:06.061820237Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D0ACAB6178, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:15:06.142180768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:15:06.144573258Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D0B19B333C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:15:07.269976927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:15:07.272484529Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D0F4D4BB24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:15:07.331833404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:15:07.335095524Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D0F890607B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:15:07.597748837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:15:07.602609296Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D108828606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:15:08.526754391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:08.891281882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:15:08.89371922Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D15576B4BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:15:09.158728334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:15:09.161271152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D1656956FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:15:10.899250641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:15:10.901795478Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D1CD278A2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:15:11.300643006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:15:11.303588234Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D1E51ADA2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:15:13.526504524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:14.845156904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:15:14.849598868Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D2B8766917, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:15:14.901839006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:15:14.904215038Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D2BBB6DF91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:15:14.904243474Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:14.904273761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:15:14.906191322Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D2BBD824FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:15:14.953250563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.3 MB" avg_comp="670 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" max_age=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:15:14.957097262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D2BEDEF87F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:15:18.526466346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:19.671368334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:15:19.676435236Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D3D82748CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:15:22.026875558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:15:22.029349093Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D46467F570, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:15:23.50869595Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:15:23.526495976Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:15:23.526497114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:23.526532115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:15:24.522681611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:15:24.525765357Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D4F935298C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:15:26.205166227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:15:26.210060601Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D55D983EA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:15:27.775495094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:15:27.778165396Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D5BB0FE795, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:15:27.778184927Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:27.778204985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:15:27.780168246Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D5BB30A573, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:15:28.526704558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:31.260258379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:15:31.262757651Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D68AC29236, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:15:32.433145539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:15:32.435893853Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D6D0AF45AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:15:32.883222991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:15:32.885782884Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D6EB7ED5F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:15:33.526338306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:36.574971493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:15:36.577509056Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D7C7898653, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:15:36.636744667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:15:36.639683678Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D7CB410EEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:15:38.526632602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:39.241757332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:15:39.246350444Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D8669EA9EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:15:39.246394372Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:39.246417679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:15:39.248421934Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D866C05B6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:15:43.495878612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:15:43.50055516Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D96431789E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:15:43.526854371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:45.022051688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:15:45.026658268Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D9BF284619, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:15:45.026682501Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:45.026708119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:15:45.028629108Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D9BF47BF5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:15:45.796755814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:15:45.799030576Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D9ED31EFF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:15:45.799048502Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:45.799070852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:15:45.800820099Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1D9ED4FABB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:15:46.577646586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:15:46.579965336Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DA1BBDF4CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:15:47.016626688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:15:47.023896531Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DA3633BD04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:15:47.033199493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:15:47.03944134Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DA37205F66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:15:47.476532576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:15:47.480973506Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DA5171978D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:15:48.526409841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:49.151246883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:15:49.153672445Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DAB524FF6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:15:49.471599395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:15:49.474385991Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DAC83F1745, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:15:49.474406931Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:49.474429681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:15:49.476837781Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DAC86800D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:15:52.598150876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:15:52.600562538Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DB82982E77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:15:52.600583529Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:15:52.600605465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:15:52.602513617Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DB82B802D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:15:53.526330431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:15:53.526341398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:53.526386513Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:15:54.931502711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:15:54.933860304Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DC0DAC9F52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:15:56.259281889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:15:56.261613434Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DC5CD03C9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:15:57.673924161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:15:57.677306725Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCB1270E14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:15:57.792845329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:15:57.79569068Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCB83FEC7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:15:57.992412683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:15:57.999031742Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCC45EC50B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:15:58.491404819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:15:58.496151651Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCE1FFB54B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:15:58.526465035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:15:58.700986283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:15:58.703323635Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCEE58B18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:15:58.707348523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:15:58.709415835Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DCEEB77E40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:15:59.138476742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:15:59.140911863Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DD086EBF1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:16:00.351563138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:16:00.354037898Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DD50BC6ADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:16:01.393473834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:16:01.395856386Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DD8ED5C6FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:16:02.706264291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:16:02.708698803Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DDDD168E71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:16:02.803938454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:16:02.806678216Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DDE2ED03E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:16:03.526017158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:07.562264785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:16:07.564843566Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DEFE88EBA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:16:08.526274433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:10.123728987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:16:10.126311063Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DF97351861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:16:10.343914616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:16:10.346268773Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1DFA452D92F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:16:12.96834647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:16:12.970747975Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E040C05FC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:16:13.52687709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:17.826316803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:16:17.829177222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E162569E19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:16:18.526564048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:19.175580866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:16:19.181977948Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E1B2F856E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:16:22.755263197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:16:22.757803666Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E2881AD862, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:16:23.508931063Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:16:23.525909536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:16:23.525955049Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:16:23.525951945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:25.136073756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:16:25.138457067Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E316025196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:16:25.511485183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:16:25.513956511Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E32C62A6F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:16:28.52614535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:29.485186314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:16:29.487824963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E4193EEA76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:16:29.538297326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:16:29.540737883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E41C66B6BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:16:29.805193887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:16:29.807860761Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E42C52BE33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:16:32.19211647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:16:32.196973471Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E4BAB9A126, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:16:33.223144944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:16:33.228826928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E4F839D25A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:16:33.228851755Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:16:33.228877514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:16:33.230740178Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E4F85A6A3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T19:16:33.52602258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:36.098989893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:16:36.10194704Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E5A37B176C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:16:36.101976349Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:16:36.102008812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:16:36.106705817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E5A3C51E39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:16:38.526140911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:39.100577038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:16:39.111049232Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E656D56445, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:16:41.167784104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:16:41.172404017Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E6D1B1E550, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:16:41.541701985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:16:41.544407212Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E6E7E006DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:16:43.526618722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:43.640104237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:16:43.642715271Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E764F200B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:16:43.642735167Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:16:43.642759255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:16:43.644846277Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E765151D2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:16:43.723073513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:16:43.725410403Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E769DFFADE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:16:45.020823828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:16:45.023090433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E7B739EE88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:16:45.102316779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=info ts=2026-06-30T19:16:45.103516787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:16:45.104625238Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E7BC15A2E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:16:45.105606895Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E7BC26BF50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:16:46.690409284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:16:46.694756943Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E81ADB4421, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:16:46.701923824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:16:46.704604706Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E81B734D5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:16:48.476407105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:16:48.480725293Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E8854F8777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:16:48.525879953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:51.072170781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:16:51.075847813Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E91FFEE888, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:16:52.740494992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:16:52.745489057Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E983824B11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:16:53.483534944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:16:53.486472349Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1E9AFAC7CB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:16:53.525886693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:16:53.525902446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:53.525923472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:16:56.506993805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:16:56.511738805Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EA63FF710E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:16:58.526435771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:16:59.14000398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:16:59.1425631Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EB00CCD938, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:16:59.572894983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:16:59.575309545Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EB1A99A120, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:16:59.772802845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:16:59.775177999Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EB26838A41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:17:00.167258328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:17:00.169767241Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EB3E08A0BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:17:03.526412244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:06.846907384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:17:06.849763793Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1ECCC316550, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:17:08.526572723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:09.609304629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:17:09.615983253Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1ED7111EEA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:17:13.526619311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:14.519162169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:17:14.523942497Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EE959BE489, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:17:17.211683497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:17:17.214051226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EF35F3EFFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:17:17.2170817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:17:17.219070116Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EF36417067, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:17:18.526043741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:18.708404706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:17:18.710788233Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EF8F29D8A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:17:19.156699983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:17:19.159194903Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1EFA9E47D04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:17:21.13817141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:17:21.140433083Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F01FFBFD97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:17:21.729477987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:17:21.736752054Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F043874E29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:17:21.73677614Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:17:21.736804036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:17:21.746868433Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F0442211A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:17:23.509554631Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:17:23.52578958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:17:23.525829466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:17:23.525838351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:23.612569436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:17:23.6174061Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F0B39EA0E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:17:24.598982591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:17:24.601539354Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F0EE47E313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:17:25.01564928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:17:25.018025973Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F1071C3E75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:17:26.201851322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:17:26.204261961Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F14DCF8B45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:17:27.395934803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:17:27.400673784Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F1951E806A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:17:28.330566491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:17:28.336806316Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F1CCEAFE2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:17:28.526591597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:30.076537219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:17:30.081331365Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F234E691FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:17:32.624758771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:17:32.627481783Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F2CCA80589, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:17:32.627504118Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:17:32.627528665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:17:32.629512922Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F2CCCAA999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:17:32.771037997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:17:32.778812613Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F2D5B05A0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:17:33.525881716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:35.599470741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:17:35.604094912Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F37E156940, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:17:37.913409001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:17:37.916205967Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F407E4379A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:17:38.526661658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:40.248699146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:17:40.251142072Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F493128091, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:17:40.251162531Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:17:40.251181722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:17:40.253041392Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F49330E849, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:17:41.946864664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:17:41.950080167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F4F856E63D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:17:43.525873396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:44.926385818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:17:44.93104995Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F5AA027C64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:17:45.345281209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:17:45.347636871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F5C2D7964C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:17:48.526219677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:49.849707196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:17:49.852497972Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F6CF5586C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:17:50.233723908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:17:50.23658685Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F6E63F9D8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:17:51.225309828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:17:51.229838216Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F7217317BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:17:53.526691409Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:17:53.526695981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:53.526729721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:17:53.535923758Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-06-30T19:17:53.538145239Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011213279s
level=info ts=2026-06-30T19:17:54.448151932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:17:54.455839209Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F7E1BC0E09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:17:54.525380982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:17:54.530253141Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F7E62CC559, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:17:56.781313185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:17:56.785761801Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F86C9C9B81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:17:57.285406488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:17:57.287840434Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F88A8A07B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:17:57.368643081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:17:57.371135735Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F88F8098CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:17:58.526035759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:17:59.26906064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:17:59.271709793Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F900C90DD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:18:01.120280447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:18:01.123298394Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1F96F2509AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:18:03.525974828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:04.874316663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:18:04.879377152Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FA4F05E036, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:18:07.915112911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:18:07.917644379Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FB041F3838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:18:08.237411329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:18:08.240143459Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FB1757AF8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:18:08.526375906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:10.990610365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:18:10.99540251Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FBBB90D5DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:18:10.995424283Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:18:10.995451839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:18:11.002151517Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FBBBFAB917, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:18:11.430148494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:18:11.434771159Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FBD5C25DF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:18:11.558024681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:18:11.560539042Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FBDD40298E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:18:11.627729522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:18:11.630205657Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FBE1680FBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:18:12.557617795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:18:12.560020629Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FC18D40F19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:18:13.18972876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:18:13.192350077Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FC3E83429B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:18:13.202557384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:18:13.206695615Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FC3F604EB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:18:13.525846254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:18:18.526458638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:20.454708391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:18:20.459511211Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FDEFAC26E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:18:22.106403951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:18:22.108733258Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FE51F9FA05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:18:23.508861625Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:18:23.525676809Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:18:23.525822835Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-06-30T19:18:23.525832004Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:18:23.525855305Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.99µs success=true
level=info ts=2026-06-30T19:18:23.525855318Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:18:23.525862473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:23.525891442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:18:24.873713218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:18:24.876938747Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FEF6F9BFEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:18:24.876960127Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:18:24.876983169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:18:24.881781692Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FEF744CFC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:18:25.543402632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:18:25.547781678Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FF1EF5AFCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:18:26.183466083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:18:26.18867897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF1FF45288F79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:18:28.52615209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:29.838223355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:18:29.842778072Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2001EF566AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:18:30.608290371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:18:30.611313713Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2004CC3E83C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:18:32.08964611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:18:32.094712366Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF200A52CDB10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:18:33.526789243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:34.532196029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:18:34.534879643Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20136A1D4DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:18:35.129468508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:18:35.131824741Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2015A37718C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:18:35.219066343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:18:35.221519918Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2015F8EB76E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:18:38.291764831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:18:38.295984505Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20216CFA8A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:18:38.526898075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:38.59313672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:18:38.597749282Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20228CD06BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:18:39.050123611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:18:39.057383975Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF202443143BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:18:43.525929026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:48.429748473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:18:48.435051622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20473253151, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:18:48.435075338Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:18:48.435102376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:18:48.43964675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF204736BFA5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:18:48.525970759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:53.502716977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:18:53.507461398Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF205A179826E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:18:53.526623083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:18:53.526667159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:18:53.526690939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:54.06481902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:18:54.067329164Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF205C2D8FE81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:18:54.245891592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:18:54.248799112Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF205CDAD7BBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:18:54.929896769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:18:54.93413456Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF205F684FAA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:18:55.428718079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:18:55.431131442Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20614255158, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:18:58.526576736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:18:59.456845167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:18:59.459217276Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF207043C995E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:19:00.796595939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:19:00.799815752Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF207542585CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:19:01.661188187Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF207877E9C52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T19:19:02.616808921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:19:02.621570711Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF207C0BA7952, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:19:03.004828854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:19:03.007378079Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF207D7B8CDDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:19:03.526859896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:03.534952403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:19:03.539654947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF207F774058E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:19:04.888939264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:19:04.893809525Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF208482A961C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:19:06.583535202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:19:06.585931463Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF208AD067947, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:19:07.073629326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:19:07.076076221Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF208CA3D0662, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:19:07.346989583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:19:07.349820615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF208DA8E105B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:19:07.780024597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:19:07.784308856Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF208F4731441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:19:08.526225312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:08.621541219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:19:08.624220313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20926822B69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:19:11.808180332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:19:11.810634546Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF209E4710102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:19:12.730231483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:19:12.732633947Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20A1B653644, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:19:12.960807494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:19:12.965063121Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20A293F072A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:19:13.52643082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:15.505470241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:19:15.510314113Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20AC0F530AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:19:18.526325587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:20.463174407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:19:20.465729659Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20BE85201FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:19:20.978270579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:19:20.981010546Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20C0709A6F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-30T19:19:20.981030326Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:19:20.981053118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:19:20.985494622Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20C074FDE9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:19:21.569803591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:19:21.574182841Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20C2A64E049, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:19:23.509080606Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:19:23.526500745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:19:23.526551078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:19:23.526549223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:19:28.526502554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:28.661779774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:19:28.664350256Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20DD0FF5AA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:19:30.111873582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:19:30.114712198Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20E2773AA59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:19:33.526505378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:34.199685898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:19:34.205775382Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20F1B4BE884, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:19:34.757317755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:19:34.764857156Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20F3C9E150E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:19:37.563603998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:19:37.568169766Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20FE3B538EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:19:37.568202627Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:19:37.568226788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:19:37.570123868Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF20FE3D50E52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:19:38.525831649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:39.485181596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:19:39.48816114Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21056255EE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:19:40.448729937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:19:40.453574473Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2108FB06A5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:19:43.526688104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:45.631853721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:19:45.634762816Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF211C4840EF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:19:48.080700907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:19:48.085745285Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF212569A46D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:19:48.526527205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:48.986736382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:19:48.989163557Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2128C73B3B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:19:49.003316877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:19:49.005580013Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2128D6E74B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:19:50.05100405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:19:50.053550137Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF212CBE50032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:19:51.630175325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:19:51.635026512Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2132A26FE21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:19:52.572153497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:19:52.577340183Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF213625212EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:19:53.16714035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:19:53.169510594Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF213859F0191, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:19:53.52671554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:19:53.526745528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:19:53.526753281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:19:58.526775737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:19:59.157302438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:19:59.160690164Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF214EAB7A549, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:19:59.20885082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:19:59.213618409Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF214EDE11477, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:19:59.268650823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:19:59.270853215Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF214F14B4683, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:20:00.060178079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:20:00.062685655Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF215207C3339, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:20:00.2011326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:20:00.203573946Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21528E2DCA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:20:02.588688912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:20:02.591159292Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF215B731D4C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:20:03.029543874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:20:03.033630361Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF215D18FFF89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:20:03.328495089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:20:03.333234115Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF215E36B909F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:20:03.526006348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:05.468836841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:20:05.471742803Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21662E4207E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:20:06.008543087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:20:06.013227193Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21683299A73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:20:07.653857274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:20:07.656541542Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF216E51D0CBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:20:07.65858511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:20:07.660745699Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF216E55EA2CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:20:08.52693661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:09.577809627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:20:09.580224083Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21757C6396F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:20:10.871236461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:20:10.874131242Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF217A4E69A2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:20:12.235933615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:20:12.240688882Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF217F6595C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:20:13.525932975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:20:18.526747474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:20.262800011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:20:20.265284498Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF219D4A7CD4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:20:23.50887342Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:20:23.526323528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:20:23.526374093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:20:23.526392902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:20:28.526828724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:28.586893001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:20:28.589560336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21BC4D175DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:20:29.313310363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:20:29.316666612Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21BF01C8E4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:20:31.551901489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:20:31.554469928Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21C7589E3D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:20:31.554490884Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:20:31.554511602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:20:31.556454566Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21C75AA475E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:20:33.526251346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:33.655798358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:20:33.665158662Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21CF359CD08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:20:35.069554954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:20:35.076367164Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21D47748E76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:20:36.053544561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:20:36.056759801Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21D81E3E836, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:20:38.526812068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:38.839332368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:20:38.846248627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21E28298BD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:20:40.043649319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:20:40.047922927Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21E6FCA204D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:20:41.550945145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:20:41.553401697Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21EC9857D5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:20:42.493588536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:20:42.496088167Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21F01B68D2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:20:43.526718659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:44.235164881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:20:44.237984078Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21F6988CD8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:20:44.238010137Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:20:44.23804372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:20:44.240286462Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21F69ADC37A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:20:46.484136378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:20:46.486599575Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF21FEF91CD6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:20:47.565100759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:20:47.567664986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2203000037D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:20:48.526415086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:48.565563956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:20:48.567876732Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2206B9E8CF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:20:49.127954113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:20:49.130497486Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2208D267255, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:20:49.671749808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:20:49.675072127Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF220AD9CCEAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:20:50.095188127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:20:50.099647002Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF220C6EC696A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:20:50.167865381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:20:50.170537653Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF220CB2687DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:20:50.724401903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:20:50.728878907Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF220EC6E0767, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:20:51.41870933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:20:51.421158012Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22115B01962, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:20:51.421179321Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:20:51.421200366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:20:51.422995157Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22115CFA49A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:20:53.525994268Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:20:53.526000426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:53.526029793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:20:53.53719136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:20:53.539985168Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22193FAA381, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:20:54.035502398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:20:54.039668526Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF221B1C45EB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:20:56.462424244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:20:56.467314019Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2224274D35E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:20:58.525993503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:20:58.79765706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:20:58.800131859Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF222CD81B085, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:20:58.80015466Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:20:58.800176935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:20:58.802086254Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF222CDA3603E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:21:03.526466893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:04.226968397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:21:04.229642654Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF224112128C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:21:05.711174702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:21:05.714268479Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22469A01908, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:21:07.77179945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:21:07.777975498Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF224E4A15D2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:21:07.930143119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:21:07.932522733Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF224EDD8C318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:21:08.526678918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:08.954914025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:21:08.957263429Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2252AEC8EEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:21:12.726715476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:21:12.729141383Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2260BBEAE4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:21:13.52598643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:15.696721593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:21:15.699075543Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF226BCC3DD1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:21:16.389377703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:21:16.392400783Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF226E61120F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:21:17.858194568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:21:17.860720399Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2273D9D1F73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:21:18.526112433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:18.986972634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:21:18.989903264Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22780EA7E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:21:22.522856104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:21:22.529674006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22853E597EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:21:23.508671467Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:21:23.522970754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:21:23.52545315Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2288F3FE16C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:21:23.526408975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:21:23.526440763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:21:23.526524509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:24.198541716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:21:24.201189578Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF228B787074F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:21:25.416346974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:21:25.418854149Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF229001C3A6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:21:26.008636969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:21:26.011170359Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2292369CA94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:21:26.140483286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:21:26.147870076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2292B9070A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:21:28.526124975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:30.991043302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:21:30.996149766Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22A4C8A357C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:21:30.996172057Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:21:30.996204749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:21:30.998208664Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22A4CACE237, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:21:31.667245917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:21:31.669478203Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22A74AE5B61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:21:33.081758141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:21:33.086495222Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22AC9231BDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:21:33.526426566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:37.309464136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:21:37.314389007Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22BC5229393, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:21:37.443554597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:21:37.4459775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22BCCFCE4DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:21:37.44599648Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:21:37.44601563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:21:37.44772129Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22BCD193BD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:21:38.526400046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:38.558562966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:21:38.561202801Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22C0F74AA0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:21:39.804857607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:21:39.807081946Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22C59B7B7F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:21:39.807101915Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:21:39.807122091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:21:39.808787361Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22C59D4ADF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:21:41.258178648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:21:41.261075175Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22CB062349A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:21:41.276429465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:21:41.281635631Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22CB19BFCEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:21:41.579583181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:21:41.58431875Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22CC3A51EE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:21:41.584341757Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:21:41.584379426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:21:41.586291123Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22CC3C69E01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:21:41.805219397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:21:41.807636768Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22CD0F58A47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:21:43.074158985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:21:43.076819573Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22D1C9A92EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:21:43.525820741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:43.916553827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:21:43.920566898Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22D4EE6CE5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:21:43.920588242Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:21:43.920612425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:21:43.926687308Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22D4F4493D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:21:44.001878034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:21:44.00631604Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22D5401E2A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:21:44.327874698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:21:44.330125335Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22D67500298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:21:48.526690264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:48.626983286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:21:48.63004882Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22E679BC2A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:21:51.602776463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:21:51.607227042Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22F190E998A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:21:53.525857315Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:21:53.52590263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:21:53.525914969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:21:55.142991206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:21:55.145656271Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF22FEBF72D01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:21:57.161113613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:21:57.163640384Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF230643F8C88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:21:58.319020263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:21:58.321412366Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF230A9409FDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:21:58.526040988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:00.232693885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:22:00.23519257Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2311B5356A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:22:01.120923071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:22:01.125075863Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF231505D9C11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:22:03.525945496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:04.267557453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:22:04.274520476Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2320C15DBD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:22:04.274547663Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:04.274574132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:22:04.276448843Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2320C36C0CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:22:04.812153092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:22:04.814547644Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2322C47DEF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:22:08.525987248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:11.915406023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:22:11.917990526Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF233D3AD6202, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:22:12.831346422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:22:12.833791783Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2340A434D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:22:13.526255454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:14.068679366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:22:14.071467377Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2345407F2A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:22:15.358229855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:22:15.364201264Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF234A115DCF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:22:15.364224593Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:15.364249196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:22:15.366056277Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF234A134E276, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:22:17.000945664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:22:17.003229888Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23502C78965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:22:17.003251226Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:17.003273447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:22:17.004998494Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23502E53CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:22:17.572407175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:22:17.574781471Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23524D8B683, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:22:18.526721985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:18.879015756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:22:18.884033367Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23572E2BE24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:22:19.167704334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:22:19.170095409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:22:19.172150544Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF235840E1B8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:22:19.172166044Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:19.172192386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:22:19.173116352Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF235841F97E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:22:19.173888408Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF235842C1DEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:22:19.944014052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:22:19.946121598Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF235B230F682, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:22:23.508851804Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:22:23.526542248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:22:23.526584147Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:22:23.526613248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:23.669294271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:22:23.671716129Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236904096B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:22:24.032743692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:22:24.035249356Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236A5EAFE87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:22:24.755418401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:22:24.758262773Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236D1021278, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:22:24.792326167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:22:24.799656542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236D37C55AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:22:25.217212258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:22:25.219651642Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236EC84A520, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:22:25.219673036Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:25.219693978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:22:25.221529268Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF236ECA36EEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:22:26.628793224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:22:26.633243889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23740C596FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:22:27.217093163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:22:27.222061794Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23763DF0C47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:22:28.500028984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:22:28.504545354Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF237B04ED136, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:22:28.526742556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:30.85599651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:22:30.85840038Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2383C9B44DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:22:31.053912968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:22:31.056281913Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF238486798B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:22:31.961449312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:22:31.963393621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:22:31.965084059Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2387E8FB053, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:22:31.967802556Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2387EBD5F16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:22:31.967818436Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:31.967839675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:22:31.969741552Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2387EDD522C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:22:32.939785716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:22:32.942233017Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF238B8D0F12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:22:33.526425989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:34.795997334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:22:34.799027736Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF239277D3605, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:22:35.614649636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:22:35.619720756Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2395867E1EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:22:36.082955962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:22:36.085232602Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23974280B75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:22:36.085252502Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:22:36.085273204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:22:36.087221915Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2397446C3AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:22:38.437479483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:22:38.439791336Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23A00802150, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:22:38.526258674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:39.7719111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:22:39.774374747Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23A500BEB6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:22:41.596986419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:22:41.599451779Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23ABCD30026, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:22:43.52678916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:45.51329882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:22:45.515665093Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23BA64143C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:22:48.526616584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:51.492562295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:22:51.495297431Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23D0AAA092C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:22:51.974742205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:22:51.977078596Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23D27629434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:22:53.525937608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:22:53.525980711Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:22:53.525975111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:53.53472138Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-06-30T19:22:53.537101233Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011116858s
level=info ts=2026-06-30T19:22:54.464705594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:22:54.467073584Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23DBBCC5E9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:22:55.646581371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:22:55.648914047Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23E023E82FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:22:58.526274766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:22:59.475123006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:22:59.477610515Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23EE672BE88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:23:00.031833064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:23:00.03676784Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23F07C77617, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:23:00.453773451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF23F20A57D63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T19:23:02.483248219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:23:02.487659319Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF23F99DC810C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:23:03.526403118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:05.148486135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:23:05.150983736Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF240389B2668, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:23:05.41072854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:23:05.413866853Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2404847046C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:23:05.413898957Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:23:05.413918946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:23:05.418611078Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2404891AD7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:23:05.72066809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:23:05.725022505Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2405AD2A5A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:23:06.463169372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:23:06.465535787Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24086F7268F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:23:08.526307487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:10.489395237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:23:10.49837594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2417756BAAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:23:11.846348559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:23:11.851077139Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF241C7F5C57F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:23:13.526519676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:13.990430335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:23:13.993322707Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24247A79680, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:23:14.380347394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:23:14.385596832Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2425F0882F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:23:14.608477586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:23:14.610738624Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2426C75BBEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:23:14.649803771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:23:14.652138128Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2426EEB31E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:23:14.941966009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:23:14.944692091Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF242805B9C1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:23:17.368487375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:23:17.371757523Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2431106953C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:23:18.227138299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:23:18.232084208Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF243444D69D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:23:18.232108166Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:23:18.232131885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:23:18.234079156Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF243446E23FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:23:18.526063764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:23.508711697Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:23:23.526106779Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:23:23.526103829Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:23:23.526138821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:23.526145508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:23:23.526266269Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-06-30T19:23:23.526274482Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:23:23.526293854Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.252µs success=true
level=info ts=2026-06-30T19:23:24.195475592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:23:24.198342232Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF244A7EB3730, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:23:25.658270083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:23:25.662793715Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF244FF36CC85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:23:26.31457889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:23:26.316955721Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2452633BCBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:23:26.326996215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:23:26.329164208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24526EE0BBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:23:26.461946847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:23:26.465833751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2452F124B9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:23:26.534617572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:23:26.539722827Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2453379C599, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:23:28.423995383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:23:28.426463355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF245A3EE084B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:23:28.52602708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:29.602271212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:23:29.605452848Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF245EA2A7814, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:23:29.605482257Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:23:29.605512718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:23:29.607539011Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF245EA571670, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:23:31.347491278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:23:31.35021165Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24652326F03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:23:31.790843264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:23:31.793701982Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2466CA3F1E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:23:33.526442596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:34.612196904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:23:34.616879091Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24714E932CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:23:36.004466654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:23:36.007093755Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24767C560E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:23:36.461868768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:23:36.464215184Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24783052810, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:23:38.526491858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:41.782621388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:23:41.785146079Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF248C02CF28A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T19:23:43.526615075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:45.632571837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:23:45.635110371Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF249A5A60E90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:23:48.526301518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:51.041900093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:23:51.046738657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24AE834B763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:23:53.525785196Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:23:53.52582177Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:23:53.52583167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:56.14525509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:23:56.151806322Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24C187E317F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:23:56.485842151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:23:56.488168642Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24C2C8B2643, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:23:57.346197217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:23:57.348835988Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24C5FD7F464, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:23:57.638763963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:23:57.641964056Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24C7151861C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:23:58.526471993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:23:58.907947588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:23:58.913134994Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24CBD14607A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:24:00.160865329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:24:00.165931249Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D07C050F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:24:01.006814304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:24:01.014037786Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D3A4E0B79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:24:01.014058825Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:24:01.014086906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:24:01.023529264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D3AE08777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:24:01.634629016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:24:01.639738393Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D5F9A1652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:24:02.597411535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:24:02.601663581Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D98F01377, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:24:02.682742641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:24:02.685075797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24D9DE830C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:24:03.52620412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:05.992374554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:24:05.994920944Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24E63304200, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:24:07.973085806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:24:07.976014332Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24ED944EBB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:24:08.525979828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:09.100844333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:24:09.103118707Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24F1C73749E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:24:12.336695708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:24:12.33925972Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24FDD564066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:24:12.459477017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:24:12.461814801Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24FE4A599D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:24:12.461834609Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:24:12.461853233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:24:12.463574015Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF24FE4C36914, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:24:13.526138918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:14.507118796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:24:14.509631518Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2505EB4085E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:24:14.94927504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:24:14.951693562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF250790E5AFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:24:15.686885262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:24:15.694172945Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF250A54E1E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:24:15.854666897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:24:15.860604666Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF250AF397FB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:24:16.582971295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:24:16.585432653Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF250DA6D5867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:24:18.373326006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:24:18.376310517Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF251452D42DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:24:18.526486713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:19.826975214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:24:19.831831246Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2519BEF4F31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:24:22.188123906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:24:22.192476798Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25228A23AAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:24:23.508658102Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:24:23.525980235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:24:23.526021153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:24:23.526019582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:27.298053902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:24:27.302678685Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF253593BB2E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:24:28.264175987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:24:28.266848777Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25392B2AFD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:24:28.266869391Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:24:28.266890954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:24:28.268883391Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25392D433C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:24:28.526173751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:29.405509165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:24:29.408163024Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF253D6B99095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:24:31.981217258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:24:31.983569662Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF254703C4959, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:24:33.525981516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:38.499521359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:24:38.502824334Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF255F4CF394F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:24:38.50285032Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:24:38.50287532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:24:38.509157757Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF255F5307FD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:24:38.526407103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:39.673478723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:24:39.678395305Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2563AE0BD30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:24:41.446604042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:24:41.449736952Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF256A475D02E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:24:43.526887322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:44.7238719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:24:44.728327442Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25767E0CACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:24:45.298684592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:24:45.303475968Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2578A297145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:24:45.303501102Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:24:45.30352738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:24:45.31086335Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2578A9C3464, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:24:45.833982533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:24:45.839129436Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF257AA14C67F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:24:46.2151763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:24:46.217830985Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF257C0A93F50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:24:48.526384766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:49.140067133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:24:49.142463083Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2586EFB1B35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:24:49.928642554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:24:49.931007571Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2589DFC84AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:24:51.996111366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:24:51.999337143Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25919451AD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:24:52.029393472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:24:52.040423231Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2591BB7E828, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:24:53.525786347Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:24:53.525830273Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:24:53.525867963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:54.500306234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:24:54.505160765Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF259AE9F848F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:24:55.370933267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:24:55.373988399Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF259E269C6B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:24:57.589085059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:24:57.593993912Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25A66BC18D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:24:57.851589139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:24:57.854060809Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25A763BAEB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:24:57.914399227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:24:57.918903862Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25A7A1A7B97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:24:58.5267361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:24:58.650117989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:24:58.654714986Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25AA5F5519F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:25:00.07216822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:25:00.074902868Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25AFA9B8421, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:25:00.900758447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:25:00.903060415Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25B2BF858BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:25:01.010340756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:25:01.013143887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25B3288CD30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:25:01.013167713Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:25:01.013191737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:25:01.01796098Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25B32D326EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:25:01.135396055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:25:01.139800147Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25B3A15CCD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:25:01.452406536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:25:01.45476218Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25B4CDB46C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-30T19:25:03.525931337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:04.053199879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:25:04.055577725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25BE7E035BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:25:04.557125214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:25:04.559867799Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25C05EE73DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:25:07.50012867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:25:07.502577558Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25CB5550B22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:25:08.526106471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:10.268474977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:25:10.270790336Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25D5A559B05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:25:13.526120253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:13.804805495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:25:13.807653407Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25E2D252CD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:25:15.351161432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:25:15.355946751Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25E896EA9E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:25:18.526419974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:18.938333649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:25:18.940757997Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25F5F1A99B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:25:20.244635981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:25:20.248862935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF25FAD110D35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:25:23.374327857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:25:23.379393272Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26067AA1193, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-30T19:25:23.379415939Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:25:23.379439267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:25:23.383375505Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26067E884E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:25:23.509201518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:25:23.526586161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:25:23.526632371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:25:23.526671498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:23.801567689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:25:23.805860999Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF260811489AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:25:25.367467828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:25:25.370721736Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF260DE5C82BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:25:27.443701716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:25:27.447672715Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2615A27CB21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:25:28.242208396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:25:28.244638375Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26189A7FEA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:25:28.526886705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:25:33.526747722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:33.928570014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:25:33.931103726Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF262DC98340C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:25:33.985184229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:25:33.987605761Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF262DFF7207A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:25:37.087216284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:25:37.092105422Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF263990096D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:25:37.412134103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:25:37.416745924Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF263AC5B2D66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:25:37.541250663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:25:37.548709603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF263B43916EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:25:37.65995004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:25:37.66469119Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF263BB22A8B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:25:38.525941821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:40.400195732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:25:40.402988487Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2645E593292, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:25:41.566297778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:25:41.570786392Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF264A3F50726, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:25:41.925983966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:25:41.928450865Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF264B945E386, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:25:43.526759422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:47.465678397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:25:47.468118592Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26603764570, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:25:48.526534147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:49.377473116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:25:49.379711556Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2667568E589, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:25:50.983721759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:25:50.986070436Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF266D525D221, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:25:51.773331355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:25:51.776245269Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF267043F8DB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:25:53.215203779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:25:53.222185944Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2675A6C73AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:25:53.336284634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:25:53.340752279Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2676180D1AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:25:53.52575758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:25:53.525795658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:25:53.525873205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:25:56.240333486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:25:56.242755088Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2680E78F498, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:25:56.242774015Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:25:56.242795995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:25:56.244685809Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2680E98A88F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:25:56.67863905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:25:56.680950918Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2682897D69F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:25:57.215407197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:25:57.218011477Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2684898CB70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:25:58.526137107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:02.851158127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:26:02.855171829Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF269989AF280, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:26:03.526441269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:03.749801462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:26:03.754692956Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF269CE360CFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:26:04.648666443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:26:04.651050098Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26A03A5ACFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:26:05.707306283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:26:05.710250782Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26A42C6F375, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:26:05.854483473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:26:05.859227118Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26A4BA7D82E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:26:08.33823833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:26:08.340799523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26ADF913BBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:26:08.526637669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:09.529100387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:26:09.53162006Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26B268D2B54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:26:10.788844619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:26:10.791262759Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26B71A1E8DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:26:13.526404282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:15.051753407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:26:15.054329939Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26C6FBABEB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:26:16.465074109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:26:16.467456763Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26CC3F54931, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:26:18.525934588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:19.964000362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:26:19.96878476Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26D94A82688, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:26:19.988815531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:26:19.995088156Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26D96381868, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:26:21.701829467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:26:21.704282812Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26DFC17B6F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:26:22.340961851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:26:22.344253247Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26E223ED0E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:26:23.508898421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:26:23.526264198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:26:23.526290589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:26:23.526325414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:24.375094616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:26:24.379955434Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26E9B94357B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:26:24.391992749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:26:24.39399861Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26E9C6C8A69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:26:26.362558719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:26:26.365014643Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF26F11E6791E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:26:28.525874038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:32.512162842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:26:32.514565355Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2708071DB5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T19:26:33.526680721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:34.182116169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:26:34.184941261Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF270E4019AD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:26:34.200043939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:26:34.209537046Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF270E578A679, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:26:36.091648571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:26:36.096125054Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27155EB5539, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:26:37.257658067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:26:37.260035543Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2719B4B5BC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:26:38.526761198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:38.914292786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:26:38.916645247Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF271FE09C415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:26:40.342801528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:26:40.345293879Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF272532F6E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:26:43.526225361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:43.983521409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:26:43.986371898Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2732C366D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:26:45.975949481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:26:45.980622641Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF273A313E255, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:26:48.09108875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:26:48.093565015Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2742105F9F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T19:26:48.52614783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:48.867265646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:26:48.869688825Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2744F4792ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:26:50.027031003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:26:50.029651316Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF274946C3441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:26:50.247181873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:26:50.249567833Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF274A187EF34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:26:52.692595272Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF2753326F104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T19:26:53.526051062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:26:53.526089244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:26:53.526102009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:26:55.46857521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:26:55.471185507Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF275D8C2D09E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:26:55.632837015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:26:55.637116834Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF275E2A70423, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:26:56.484474603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:26:56.489170922Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF276157089CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:26:57.803132803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:26:57.805408495Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27663E56ADD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:26:58.526743302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:27:03.52584046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:05.643077852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:27:05.645874539Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2783739DE58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:27:06.697275644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:27:06.702108485Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF278762D361D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:27:06.702136629Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:27:06.70216512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:27:06.704219632Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF278764FE56E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:27:07.78721794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:27:07.789917888Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF278B7041B18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:27:08.391494102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:27:08.396169333Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF278DB271AC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:27:08.525859088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:08.747373198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:27:08.752257783Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF278F05FA6FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:27:09.945995865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:27:09.948485734Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27937AD7715, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:27:10.018688059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:27:10.021166613Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2793C02837F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:27:11.12793957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:27:11.131211435Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2797E2B8625, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:27:13.525889841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:14.041398318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:27:14.04595979Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27A2BE76428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:27:15.300759045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:27:15.303629228Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27A76DEFAFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:27:16.073758746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:27:16.078346248Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27AA50B2C95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:27:18.526417555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:19.694365913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:27:19.697002212Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27B7CBB2C1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:27:20.21659564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:27:20.219079063Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27B9BD9530A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:27:21.628987603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:27:21.631439748Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27BF00770AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:27:22.376314415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:27:22.378837338Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27C1C9524F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:27:23.509283111Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T19:27:23.526703708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:23.526723565Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:27:23.526753696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:27:24.046924678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:27:24.055569935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27C8085F4C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:27:27.597634231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:27:27.603490844Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27D53FECDE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:27:28.129291677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:27:28.13203535Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27D737F8397, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:27:28.39065089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:27:28.393100056Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27D830F6CDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:27:28.526049783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:29.039914327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:27:29.042400488Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27DA9C3C4D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:27:31.842322029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:27:31.845306967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27E50D3E899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:27:33.526809515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:33.736191812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:27:33.742810472Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27EC1EDF895, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:27:33.74283561Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:27:33.742860897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:27:33.74475285Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27EC20E62D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:27:33.971349277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:27:33.973914706Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27ECFB4E841, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:27:37.899279857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:27:37.902150511Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27FB9D85A79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:27:37.902170182Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:27:37.902193895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:27:37.904374437Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27FB9FC8CB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:27:37.993489506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:27:37.996180619Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF27FBF72A147, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:27:38.52600753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:27:43.526410498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:43.85573327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:27:43.858224705Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2811CDA119E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:27:43.858246699Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:27:43.858269448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:27:43.860289124Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2811CFC7F8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:27:45.500178678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:27:45.504285079Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2817EF82C3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:27:48.526475011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:49.505877457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:27:49.512112383Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2826DD9ABDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:27:50.472945745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:27:50.475867605Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF282A74C802D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:27:52.433287079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:27:52.438133841Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2831C3F67B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:27:52.712518017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:27:52.715750476Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2832CCEB486, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:27:53.526103598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:27:53.526107823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:53.526139074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:27:53.53065193Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-06-30T19:27:53.533068767Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006616954s
level=info ts=2026-06-30T19:27:54.454494643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:27:54.459053213Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28394B5E88E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:27:55.192219577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:27:55.194636603Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF283C090072E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:27:56.804973875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:27:56.807331191Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28420AF5538, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:27:58.526628695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:27:58.608995653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:27:58.61176798Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2848C3A526B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:27:59.854497987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:27:59.857480091Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF284D67CA796, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:28:00.928973275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:28:00.934198585Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28516A7C553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:28:02.900340152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:28:02.902875803Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2858C00CBC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:28:02.902899621Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:02.902923927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:28:02.904836996Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2858C2155C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:28:03.526642584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:06.22577757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:28:06.228293841Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28652369673, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:28:07.26696245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:28:07.269567109Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF286904713CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:28:07.269591332Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:07.26961301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:28:07.271597661Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2869068AECE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:28:07.641884898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:28:07.646663194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF286A6C1F54E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:28:08.526307127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:11.469634388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:28:11.475920252Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2878AFED611, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:28:12.521138716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:28:12.523543208Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF287C9716527, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:28:12.523562772Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:12.523584612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:28:12.525440162Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF287C98FE00A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:28:13.526587319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:13.796095536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:28:13.798375582Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF288156E3927, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:28:14.901737861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:28:14.904433896Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF288575B54D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:28:14.946482287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:28:14.950952926Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2885A20A761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:28:17.560029925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:28:17.562535165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF288F5CA7C64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:28:18.100115194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:28:18.102516352Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28915F9DE45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:28:18.337162375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:28:18.339629404Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF289241BC660, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:28:18.526383504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:18.83412949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:28:18.836534179Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28941BA58F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:28:18.836553429Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:18.836576575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:28:18.838399444Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28941D93BE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:28:18.986477825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:28:18.989035382Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2894AD1599F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:28:19.00534505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:28:19.007577833Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2894BED358D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:28:21.283828634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:28:21.286297954Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF289D3BDD60E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:28:21.702210758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:28:21.704711098Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF289ECAE3C4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:28:21.971338391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:28:21.978680313Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF289FD037BB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:28:23.508694827Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:28:23.526125777Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-30T19:28:23.526151297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:28:23.52617919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:23.526185898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:28:23.526274868Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-06-30T19:28:23.526282898Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:28:23.526300229Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.849µs success=true
level=info ts=2026-06-30T19:28:26.815578948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:28:26.820037254Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28B1D9503AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:28:26.974074103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:28:26.976705117Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28B26EAF559, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:28:28.526719594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:29.038985737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:28:29.04178443Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28BA20067F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:28:29.252375362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:28:29.254984313Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28BAEB638AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:28:30.19200919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:28:30.194402036Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28BE6B39CBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:28:33.525829716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:36.03194716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:28:36.034773597Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28D42D235CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:28:38.526936743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:40.91180361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:28:40.920388484Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28E660516BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:28:40.920412901Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:40.920439908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:28:40.92587635Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28E665AD215, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:28:43.526135018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:45.676342236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:28:45.681096994Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28F81C85979, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:28:46.801568392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:28:46.804816873Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF28FC4C397C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:28:48.208624488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:28:48.215930047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29018DE4248, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:28:48.526024741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:51.927307486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:28:51.932063713Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF290F65F369D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:28:52.111651518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:28:52.114018333Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2910138F24D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:28:52.463904171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:28:52.466454141Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF291163A1A01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:28:52.584974133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:28:52.587296768Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2911D6E3ACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:28:52.693476839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:28:52.696080889Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29123E8EBE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:28:53.41805128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:28:53.420465437Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2914F169838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:28:53.420483852Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:53.420504989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:28:53.422339757Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2914F364AF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:28:53.526088381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:28:53.526128925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:28:53.526165111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:54.257583234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:28:54.259860164Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF291811FC14C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:28:54.64100641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:28:54.643320226Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29197FA0420, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:28:54.643340155Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:54.643380922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:28:54.645154199Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2919817A099, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:28:56.450118865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:28:56.452686367Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29203D2E915, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:28:56.452707079Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:56.452727597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:28:56.454549169Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29203F2912A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:28:56.637593896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:28:56.640771437Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2920F07448B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:28:57.238768956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:28:57.243561919Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29232F71EC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:28:57.243584705Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:28:57.243608262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:28:57.245502357Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29233173BE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:28:58.526572595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:28:58.910598304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:28:58.913015673Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2929677EA98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:28:59.031253866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:28:59.035482505Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2929DC4D2CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:28:59.643948684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:28:59.648270085Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF292C24B1345, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:29:00.901142778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:29:00.903708975Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2930D1FE129, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:29:03.526326743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:04.408511155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:29:04.410959258Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF293DE2C1FE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:29:07.424941349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:29:07.427468287Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29491F885B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:29:07.871722327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:29:07.874416438Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF294AC9A9E8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:29:08.526610664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:08.566734012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:29:08.56984721Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF294D60D59BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:29:09.659951202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:29:09.664735772Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF295175157DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:29:11.42523344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:29:11.427810001Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29580697A87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:29:11.427828568Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:11.427849295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:29:11.429664885Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2958087FEEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:29:13.526226163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:13.571372559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:29:13.573857577Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2960052B28B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:29:13.573877014Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:13.573898427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:29:13.575642816Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2960070DDAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:29:13.982870277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:29:13.985405924Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29618DA4149, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:29:15.278391143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:29:15.280819241Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2966610F4F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:29:17.926777588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:29:17.934826692Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2970441CE7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:29:18.526579853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:22.130778639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:29:22.137207627Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF297FEBD92AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:29:23.508685129Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:29:23.526415727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:29:23.526451613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:29:23.526480815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:23.695725629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:29:23.700026097Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2985BE34BCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:29:25.477987777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:29:25.480607398Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF298C6042479, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:29:26.758177666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:29:26.760882979Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29912542952, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:29:26.760901287Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:26.760922088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:29:26.762996711Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29912766439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:29:27.177677441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:29:27.181137811Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2992B61440A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:29:27.724383343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:29:27.729376204Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2994C0D0ABE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:29:28.526160123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:29:33.526711912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:34.599994311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:29:34.602552222Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29AE5BAB63D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:29:34.60257474Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:34.602595077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:29:34.6044463Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29AE5DA0F78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:29:35.943317579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:29:35.945741729Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29B35CB6E6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:29:36.310962415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:29:36.315110377Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29B4BCF0671, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:29:37.765160693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:29:37.770116126Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29BA287AD1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:29:37.770147564Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:37.770171864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:29:37.772344369Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29BA2AB2FDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:29:38.526555454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:40.705790831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:29:40.710553154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29C51CBDF93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:29:41.175746019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:29:41.180397979Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29C6DCCF01D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:29:41.783888154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:29:41.786684299Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29C91EE1611, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:29:42.255234294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:29:42.258218188Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29CAE0B7451, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:29:42.258248794Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:29:42.258282866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:29:42.263124932Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29CAE570CA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:29:43.526319144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:43.591406075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:29:43.596267818Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29CFDCBCE07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:29:44.343879809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:29:44.346278518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29D2A81325C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:29:45.042475311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:29:45.04477244Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29D5423D8B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:29:45.053812935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:29:45.055980275Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29D54CEDA3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:29:45.480216246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:29:45.482647464Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29D6E3D0D15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:29:46.385910903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:29:46.38840018Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29DA438B4B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:29:46.545531745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:29:46.548011962Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29DADBBF588, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:29:47.422917051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:29:47.425627145Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29DE20A41FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:29:48.437729965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:29:48.440344119Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29E1E85EA62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:29:48.526801279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:49.740406518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:29:49.742992168Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29E6C2B8470, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:29:50.299101227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:29:50.301913463Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29E8D7D2FDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:29:50.593953207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:29:50.598801768Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29E9F2DBCCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:29:53.484938105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:29:53.487598382Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29F4B5BDDBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:29:53.526058749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:29:53.526097637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:29:53.526134282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:56.092280487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:29:56.094883474Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF29FE6C50C4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:29:57.163403152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:29:57.166310042Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A026A19E88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:29:57.541617364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:29:57.546601905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A03D4D045E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:29:58.476195739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:29:58.478649391Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A074D9D06F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:29:58.52586436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:29:58.705344708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:29:58.7077548Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A08283E3E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:30:03.526508941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:07.154405669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:30:07.157110657Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A27A217F57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:30:08.525845623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:09.272175036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:30:09.274705706Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A2F859186F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:30:09.494249166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:30:09.501870981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A305E4898E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:30:13.526864162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:30:18.526132716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:18.564333614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:30:18.568982789Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A522553709, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:30:18.576016669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:30:18.578808383Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A522EBD80A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:30:19.003323199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:30:19.008473038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A53C871980, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:30:19.728985888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:30:19.733811749Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A567C30D35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:30:23.508676872Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:30:23.526387387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:30:23.526421054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:30:23.52640802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:23.963213411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:30:23.966463591Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A6640CA897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:30:24.788029416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:30:24.79519068Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A69570E1B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:30:27.70529854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:30:27.709786193Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A7432A8EFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:30:27.722815773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:30:27.7251672Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A74415B696, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:30:28.526652704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:32.645275166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:30:32.64788472Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8697FBD6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:30:33.525828112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:33.81046513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:30:33.812953823Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8AEF1D75A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:30:33.912469894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:30:33.914872699Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8B5053905, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:30:34.298824374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:30:34.301646781Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8CC130C6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:30:34.533541521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:30:34.538118587Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8DA28C930, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:30:34.766800567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:30:34.770735194Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8E809364A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:30:35.000471194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:30:35.004929185Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A8F5FD3A16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:30:35.265581572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:30:35.26807455Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A905ACF315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:30:38.221463186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:30:38.223883464Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A9B5DBA998, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-30T19:30:38.223902489Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:30:38.223932487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:30:38.226427404Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2A9B5F98972, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:30:38.526561238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:39.636117562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:30:39.63859992Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AA0A2CE624, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:30:42.176222457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:30:42.178769807Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AAA1956378, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:30:43.525905853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:45.088057429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:30:45.090519063Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AB4F2432FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:30:45.090539186Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:30:45.090559464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:30:45.092439935Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AB4F437EFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:30:48.394371091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:30:48.397094942Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AC14393E45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:30:48.526880765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:48.682267903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:30:48.684688551Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AC255E9F38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:30:49.124874646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:30:49.12751658Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AC3FC2F415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:30:51.095210801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:30:51.097734336Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2ACB53336AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:30:52.541116016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:30:52.543718041Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AD0B613E41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:30:53.488080763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:30:53.490802569Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AD43D586CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:30:53.526325285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:30:53.526378703Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:30:53.526379852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:30:53.784257743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:30:53.786584248Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AD55782537, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:30:53.795628228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:30:53.802661362Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AD566CDFD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:30:53.802686538Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:30:53.802715159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:30:53.813236935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AD570F131B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:30:58.526615944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:03.048934364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:31:03.054060826Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AF7DDAE933, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-30T19:31:03.526057139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:04.469501565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:31:04.474157596Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2AFD27F3001, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:31:08.526754058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:10.42885958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:31:10.431406081Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B1359354CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:31:11.02233032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:31:11.030258417Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B159468107, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:31:12.396045803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:31:12.400808855Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B1AAF670E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:31:13.327101616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:31:13.331672131Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B1E272A602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:31:13.33169567Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:31:13.331724289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:31:13.338418014Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B1E2D9B9EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:31:13.526259715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T19:31:13.946418414Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF2B20718FA80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T19:31:14.862937396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:31:14.867996192Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B23E020B24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:31:15.769877298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:31:15.774169312Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B2740840E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:31:17.415140828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:31:17.417675902Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B2D5FD7EC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:31:17.631332214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:31:17.633764108Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B2E2DFFB00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:31:18.28557158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:31:18.288062497Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B309DEBDB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:31:18.526073444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:19.756163444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:31:19.759181367Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B3618E3727, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:31:19.759201777Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:31:19.759227683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:31:19.764008703Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B361D969A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:31:21.468304068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:31:21.474337475Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B3C7C744B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:31:21.486403138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:31:21.492001576Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B3C8D29C50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:31:21.492027813Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:31:21.492056568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:31:21.496547672Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B3C91D9B3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:31:22.033221102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:31:22.037650065Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B3E95D3037, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:31:23.508812059Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:31:23.526192962Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:31:23.526228813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:31:23.526232301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:24.015525327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:31:24.020377002Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B45F882FE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:31:24.116410839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:31:24.121135365Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B4658C1ED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-30T19:31:28.526650937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:28.854897505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:31:28.8573103Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B57FD79C4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:31:29.212311548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:31:29.214869991Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B59528A55B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:31:29.25294195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:31:29.255483804Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B597947DB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:31:31.124950064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:31:31.127986722Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B60730D084, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:31:33.52662565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:34.00348493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:31:34.008566029Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B6B2DFDD94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:31:34.406905437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:31:34.409418146Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B6CAC5E303, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:31:34.45042298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:31:34.452946862Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B6CD5EF33D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:31:36.262408482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:31:36.264872185Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B7395F2EB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:31:36.320402942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:31:36.322823863Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B73CD3903F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:31:38.526439028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:40.621246532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:31:40.624761096Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B83D3D3648, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:31:41.887277254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:31:41.892178923Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B888C8C958, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:31:41.963468216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:31:41.966154726Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B88D31A51D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:31:43.044789287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:31:43.526118805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-30T19:31:44.606215401Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B92A8D8DAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:31:45.484959344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:31:45.489723194Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B95F371288, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:31:47.610588518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:31:47.617668393Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B9DE0CFE0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:31:47.821173208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:31:47.825712816Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2B9EA736729, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:31:48.526756288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:49.953801483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:31:49.956425445Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BA6971E7F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:31:53.525767396Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:31:53.525803675Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:31:53.525837251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:31:54.103894942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:31:54.10673528Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BB60D56BCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:31:56.476100123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:31:56.481077973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BBEE58731A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-30T19:31:58.526651436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:00.693134874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:32:00.695876104Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BCE9926106, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:32:03.240592876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:32:03.243103208Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BD816719F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:32:03.526599863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:04.831155941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:32:04.833584507Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BDE03357CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:32:04.926052119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:32:04.93038885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BDE5F8154D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:32:05.756891478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:32:05.761807498Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BE1787503A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-30T19:32:05.761827989Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:32:05.761851003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:32:05.765612782Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BE17C3A4AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:32:06.589645553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:32:06.593926589Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BE49201F04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:32:06.593951973Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:32:06.593978105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:32:06.59582708Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BE493FA92C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:32:08.526096191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:09.89174839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:32:09.894158414Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BF0DD636E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:32:11.67273781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:32:11.675105987Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BF77FBC65E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:32:12.276609678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:32:12.279041417Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BF9BFC4F15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:32:12.618300035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:32:12.621884638Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2BFB06B5397, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:32:13.526682912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:16.00354258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:32:16.009922912Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C07A5C4468, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:32:18.526615752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:18.72420983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:32:18.728485045Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C11C66981D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:32:21.303128107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:32:21.305912051Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C1B605FC18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:32:23.508666356Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:32:23.526303724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:32:23.526307287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:23.526338773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:32:24.049496021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:32:24.052141378Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C259B68471, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:32:24.061187864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:32:24.063343612Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C25A632480, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:32:24.245255385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:32:24.247800217Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C26560E589, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:32:24.335172864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:32:24.337505372Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C26AB9C081, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:32:24.514216739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:32:24.516810969Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C2756A19AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:32:24.994002263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:32:25.001317388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C29248F7CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:32:28.183112504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:32:28.188004539Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C3503AB3E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T19:32:28.52639313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:31.657871222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:32:31.660302052Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C41F31C4C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:32:33.526675137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:34.151174343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:32:34.153772036Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C4B3CFFE1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:32:35.175911492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:32:35.178639111Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C4F0E6D8D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:32:35.387227146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:32:35.39013083Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C4FD8306BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:32:38.238193035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:32:38.242988416Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C5A78CC631, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-06-30T19:32:38.526053942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:38.538097788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:32:38.542776878Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C5B96D7BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:32:42.797478582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:32:42.801697608Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C6B746383C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:32:43.42470383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:32:43.427039694Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C6DC8BD2B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:32:43.52662711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:46.1064731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:32:46.108827967Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C77C65B982, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:32:46.365640362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:32:46.369751827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C78BF2284A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:32:46.762629674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:32:46.767384482Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C7A3A5B231, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:32:47.382314195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:32:47.384961853Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C7C87656BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:32:48.526420677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:50.069179212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:32:50.074152763Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C868C014D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:32:53.525969843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:32:53.526005083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:32:53.526028436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:32:53.533298095Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-06-30T19:32:53.535904986Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009429146s
level=info ts=2026-06-30T19:32:56.862025853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="653 B" avg_comp="653 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:32:56.866784374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2C9FD9D75AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:32:58.526563252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:00.318970755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:33:00.321560724Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CACB8A676B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:33:00.991392017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:33:00.994307752Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CAF3A5B5E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:33:03.201325809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:33:03.206302491Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CB777AF125, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-30T19:33:03.526777378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:03.705639674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:33:03.709897421Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CB9580AA8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:33:04.622314535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:33:04.624687907Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CBCC075083, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:33:06.021028375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:33:06.023464526Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CC1F668F43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:33:07.045653079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:33:07.048378996Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CC5C7852FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:33:08.525937618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:08.833655762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:33:08.835979996Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CCC70A2E0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:33:11.70746036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:33:11.714727394Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CD72A03842, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T19:33:13.526391671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:13.689745937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:33:13.696563131Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CDE8BD3A97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:33:15.802003651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:33:15.804613851Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CE6664F23B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T19:33:18.526789644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:20.346827455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:33:20.34937678Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CF7548E511, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:33:20.637716805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:33:20.640012687Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CF869D4022, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:33:20.977925015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:33:20.980762314Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CF9AECCA7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:33:22.14930634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:33:22.156266876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2CFE0FDB8AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:33:23.508666641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:33:23.526189112Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-30T19:33:23.526195996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:23.526191544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:33:23.526225708Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:33:23.52630194Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-06-30T19:33:23.526308969Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-30T19:33:23.52632639Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.843µs success=true
level=info ts=2026-06-30T19:33:24.613832695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:33:24.618859374Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D073C55C38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:33:26.171121461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:33:26.173597966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D0D0712E01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:33:28.526231664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:30.638693892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:33:30.641213035Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D1DABBB0DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:33:33.525910497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:33.565000471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:33:33.567565047Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D289289D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:33:34.24670276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:33:34.249736629Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D2B1D224EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:33:35.971734052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:33:35.977647865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D318CF86A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:33:36.124215635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:33:36.126672312Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D321B0C54F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:33:37.387558301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:33:37.389833384Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D36CFCD34C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T19:33:38.526124608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:42.024576528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:33:42.027420164Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D48167EDEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:33:42.635231334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:33:42.639980516Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D4A5EAF77C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:33:42.878542143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:33:42.880824187Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D4B4466C89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:33:43.526274863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:44.41465402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:33:44.417344538Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D50FDB4398, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:33:45.859837545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:33:45.86249666Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D565FCF43B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:33:46.222643299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:33:46.226646767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D57BB3E477, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:33:48.526920584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:49.84210043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:33:49.846762862Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D65378FC93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:33:52.212127777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:33:52.214610637Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D6E09B1239, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:33:53.30478043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:33:53.307202366Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D721BB2C20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:33:53.526219505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:33:53.526221637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:33:53.526259403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:33:56.076087945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:33:56.078517684Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D7C6E949C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-30T19:33:56.078538093Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:33:56.078558062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:33:56.080448444Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D7C7098865, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:33:57.349030618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:33:57.351439285Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D812C93DCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:33:57.484677786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:33:57.488010506Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D81AEC91AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:33:57.532852043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:33:57.537490597Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D81DDFEA8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:33:58.526687512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:34:03.526543579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:04.112618899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:34:04.120242946Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D9A63B6317, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:34:04.909697532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:34:04.91406322Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D9D58E37D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:34:05.124608167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:34:05.126951792Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D9E23E7FFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:34:05.126972687Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:05.12699344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:34:05.128730913Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2D9E25CED90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:34:08.412859147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:34:08.415300698Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DAA63DC41B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:34:08.52669529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:08.56679275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:34:08.569656863Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DAAF72B7AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:34:08.702895264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.6 kB" avg_comp="9.6 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:34:08.707541149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DAB7AA1FB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:34:12.696847649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:34:12.701627477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DBA5BBBC32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T19:34:13.52645987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:13.554570015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:34:13.559569081Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DBD8DD8D36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:34:13.559594012Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:13.55961994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:34:13.561716788Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DBD9019557, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:34:17.214646443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:34:17.217313133Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DCB2E1DCA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T19:34:18.526311224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:20.042728263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:34:20.045231818Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DD5B715DB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:34:20.104632136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:34:20.10698016Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DD5F207B79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:34:20.674602471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:34:20.676912469Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DD8118F2E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:34:22.704310842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:34:22.706568744Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DDFA13D1BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:34:22.706585883Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:22.706604837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:34:22.708324496Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DDFA308FDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:34:23.508788018Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:34:23.526349187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:34:23.526397678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:34:23.526403468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:25.259623126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:34:25.262625727Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DE92688394, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:34:25.589747895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:34:25.592317206Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DEA6133D7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:34:25.592335083Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:25.59237202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:34:25.594253499Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DEA6335BB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:34:27.731637799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:34:27.734518496Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DF25C2FC70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:34:28.397383411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:34:28.402542099Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2DF4D934DAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:34:28.526163739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:32.252301178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:34:32.254729117Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0332FB9C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:34:32.761390336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:34:32.763801839Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E051886C00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:34:33.526777412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:33.668058846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="398 B" avg_comp="398 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:34:33.670651473Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E087949269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:34:33.90232912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:34:33.904935051Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0958BF99F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:34:34.087080194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:34:34.089555362Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0A08C995B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:34:34.197083656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:34:34.199398111Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0A7192E9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:34:35.19020749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:34:35.192693175Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0E24D8F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:34:35.192714739Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:35.192737017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:34:35.194584816Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E0E26C40BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:34:36.642103998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:34:36.644927823Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E138DD6752, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T19:34:37.935294192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:34:37.941534374Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E18623967E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:34:38.526242993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:40.75956118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:34:40.764390801Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E22E64E9D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:34:41.592366458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:34:41.59480169Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E25FE55FD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:34:42.287986762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:34:42.290331194Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E2895A3E3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:34:43.526658817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:46.504031971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:34:46.506308978Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E384A4502F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:34:48.526923596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:48.687370832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:34:48.690105648Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E406CC97C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:34:48.690122403Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:48.690141541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:34:48.692036092Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E406EEDD7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:34:49.104820375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:34:49.107652364Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E41FAF7469, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:34:49.107675629Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:34:49.10770312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:34:49.109774541Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E41FD44C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:34:49.308468007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:34:49.310816792Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E42BCF06BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:34:50.260652138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:34:50.263054626Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E4649098F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:34:52.653587921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:34:52.656512948Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E4F33920C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:34:53.526202056Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T19:34:53.526221368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:53.526245559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T19:34:54.809298421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:34:54.815005946Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E573E1C840, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T19:34:57.179278352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:34:57.182560475Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E600FF4EF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T19:34:58.526636954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:34:58.54478534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:34:58.54917498Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E652752C9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T19:35:01.247075428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:35:01.252218534Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E6F390F10D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:35:02.341280206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:35:02.346247544Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E734C6E811, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T19:35:02.569973927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:35:02.577176023Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E7428BB083, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:35:02.792763369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:35:02.797052138Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E74FA52C64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:35:03.39769857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:35:03.400231496Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E77399A5C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T19:35:03.526835185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:06.473276446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:06.475670996Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E82AE97294, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:06.475692743Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:06.475713668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:06.477536758Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E82B08C3D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:35:08.344479665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:35:08.346968299Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E89A725B4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:35:08.449187917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:35:08.451630228Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E8A0B0A2C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T19:35:08.526325606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:12.187377035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:35:12.190246456Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E97F86EA18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T19:35:13.526615281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:13.676961198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:35:13.683507196Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2E9D8888B26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:35:14.386453743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:35:14.388746211Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EA02923235, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:35:14.388763475Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:14.388783151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:35:14.390482419Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EA02AF92D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:35:14.557284314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:35:14.559647118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EA0CC245CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:35:14.559667674Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:14.688842637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:35:14.691200412Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EA1498BCDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:35:15.551091239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:35:15.553566382Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EA47FECF81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T19:35:18.526560636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:18.576753738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:18.579262177Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EAFC575162, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:19.115975778Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDF2EB1C5851DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-30T19:35:20.809557575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:35:20.813704017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EB81868FEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:35:20.813728964Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:20.813756438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:35:20.81959318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EB81E28B29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:35:21.478893118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:35:21.483877503Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EBA97795CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:35:22.198742495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:35:22.20133405Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EBD43C0C16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:35:23.335815089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:35:23.338210423Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EC17FF1431, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:35:23.508738963Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:35:23.526542863Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:35:23.52658617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:35:23.526667421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:26.835402905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:35:26.838048433Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2ECE8991F79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T19:35:27.537849872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:35:27.540920588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2ED127F2A07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:35:28.525840522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:28.919132401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:35:28.924213114Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2ED64F2DEBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:35:30.19804462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:35:30.200340924Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EDB103C5B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:35:30.200374442Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:30.20039422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:35:30.202135057Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EDB12098C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:35:30.628588918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:35:30.630820482Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EDCAAD7D34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:35:31.348477654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:35:31.350787399Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EDF59693AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:35:31.886734333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:35:31.889027414Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EE15AB9F2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:35:33.062493231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:35:33.064766947Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EE5BBFA5FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:35:33.174151386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:35:33.178009921Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2EE627F9804, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T19:35:33.526774456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T19:35:38.526287808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:43.119489761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:35:43.125416144Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F0B3682897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-30T19:35:43.525982261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:47.028331902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:35:47.031211032Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F19C3671D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:35:47.031231448Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:47.031254768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:35:47.035964765Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F19C80513B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:35:47.477189053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="9.7 kB" avg_comp="9.7 kB" total_uncomp="82 kB" avg_uncomp="82 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-30T19:35:47.481936899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F1B71385AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-30T19:35:47.742961284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="172 kB" avg_comp="172 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T19:35:47.745749197Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F1C6CC3D07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-30T19:35:48.525891685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:49.664479035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.8 kB" avg_comp="4.8 kB" total_uncomp="19 kB" avg_uncomp="19 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:35:49.666935472Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F2394ED7F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T19:35:49.666956607Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:49.666979624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:35:49.668792368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F2396E941E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:35:49.848409583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:35:49.85069732Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F244441A49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T19:35:50.322972391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:35:50.325537436Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F260912B37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:35:50.325558309Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:35:50.325579017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:35:50.32768837Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F260B3B922, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:35:50.471999278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="633 kB" avg_comp="633 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T19:35:50.475165332Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F2697D01D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T19:35:51.35963697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="15 MB" avg_comp="3.0 MB" total_uncomp="133 MB" avg_uncomp="26 MB" full=3 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T19:35:51.369648319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F29ECBF5F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T19:35:53.526575579Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:35:53.526618115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:35:53.526617179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:54.509335937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T19:35:54.516468502Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F35A5A8FC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T19:35:56.231545104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="580 kB" avg_uncomp="580 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T19:35:56.236084177Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F3C0DBECD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T19:35:56.525594774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T19:35:56.52803455Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F3D243B281, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-30T19:35:57.662682328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T19:35:57.665054459Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F416093C2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T19:35:58.526389753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:35:59.812634378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:35:59.815495849Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F49636BC1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-30T19:36:00.465392165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:36:00.470210008Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F4BD3C48C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:36:00.608845025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="671 B" avg_comp="671 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T19:36:00.611179179Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F4C5A3EF06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T19:36:01.938787465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T19:36:01.941270855Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F514EB0282, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T19:36:02.709734283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="331 kB" avg_comp="331 kB" total_uncomp="1.7 MB" avg_uncomp="1.7 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T19:36:02.712472236Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F542E320F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T19:36:02.729468774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:36:02.733810328Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F5442863D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:36:02.733829644Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:36:02.733851666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:36:02.735653204Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F5444792DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-30T19:36:03.52641795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:04.138939735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="685 kB" total_uncomp="7.5 MB" avg_uncomp="3.8 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T19:36:04.14314542Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F598295AF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T19:36:04.786986636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:36:04.791369628Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F5BECC476A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:36:04.791392305Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:36:04.791415314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:36:04.793300226Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F5BEEBF465, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:36:08.077253293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.1 kB" avg_comp="4.1 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T19:36:08.079723263Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F682CC107C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-30T19:36:08.526172719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:09.850786047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="112 kB" avg_uncomp="112 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-30T19:36:09.853155356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F6EC8027DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-30T19:36:09.898235534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:36:09.900482972Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F6EF5407CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:36:09.900496394Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T19:36:09.900514287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:36:09.902196758Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F6EF7044D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T19:36:10.786177713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="140 kB" avg_uncomp="140 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T19:36:10.788583167Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F7244323D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T19:36:12.171197081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="696 kB" avg_uncomp="696 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T19:36:12.173623387Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F776D0D23A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T19:36:12.18165639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="345 kB" avg_comp="345 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T19:36:12.184106424Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F77771CDEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T19:36:13.526441264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:16.578671477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="4.2 kB" avg_comp="2.1 kB" total_uncomp="25 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T19:36:16.585750939Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F87DCBC84F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T19:36:18.077330516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="416 B" avg_comp="416 B" total_uncomp="451 B" avg_uncomp="451 B" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T19:36:18.07988094Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F8D6DA9816, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T19:36:18.526474835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:20.017657134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="139 kB" avg_uncomp="139 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T19:36:20.020317458Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F94A827A9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T19:36:22.073913349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="42 kB" avg_comp="42 kB" total_uncomp="277 kB" avg_uncomp="277 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T19:36:22.076659951Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F9C5130B99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T19:36:22.625115628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="493 kB" avg_uncomp="493 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-06-30T19:36:22.62767406Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2F9E5EBE0D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-06-30T19:36:23.241337713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="109 kB" avg_uncomp="109 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T19:36:23.24377599Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2FA0AA54D19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T19:36:23.509063155Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T19:36:23.52644193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T19:36:23.526478095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T19:36:23.526550673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:23.95067452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T19:36:23.953172966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2FA34ECD5B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T19:36:28.52599346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T19:36:28.596144973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-30T19:36:28.598678995Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2FB49D1731A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-30T19:36:29.563658252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T19:36:29.566081557Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2FB837B726D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T19:36:30.260184956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-30T19:36:30.263181041Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDF2FBAD07E1F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"