level=info ts=2026-06-29T22:19:44.048518037Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-06-29T22:19:44.048585899Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-06-29T22:19:44.050473926Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=error ts=2026-06-29T22:19:44.054254439Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T22:19:44.055533892Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:19:44.055533019Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-06-29T22:19:44.056219632Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-06-29T22:19:44.056315385Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-06-29T22:19:44.056358369Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-06-29T22:19:44.057592467Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-27acb9bd
level=info ts=2026-06-29T22:19:44.057830277Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-06-29T22:19:44.061346279Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-06-29T22:19:44.061357443Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T22:19:44.061365701Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=3.536756ms
level=error ts=2026-06-29T22:19:44.061373872Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-06-29T22:19:44.06138195Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
ts=2026-06-29T22:19:44.063684216Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:44.06369605Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-06-29T22:19:44.073210356Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-06-29T22:19:44.073212709Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-06-29T22:19:44.073561172Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-06-29T22:19:44.073586143Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-06-29T22:19:44.073601236Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-06-29T22:19:44.073651435Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-06-29T22:19:44.073690004Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-06-29T22:19:44.073809912Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-06-29T22:19:44.073825285Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-06-29T22:19:44.073836525Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:19:44.073880845Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.28µs success=true
level=info ts=2026-06-29T22:19:44.073894363Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-06-29T22:19:44.073951317Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=127.447µs errors=false
level=info ts=2026-06-29T22:19:44.073965603Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-06-29T22:19:44.07408752Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=264.411µs errors=false
level=info ts=2026-06-29T22:19:44.074101608Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-06-29T22:19:44.074109874Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=286.372µs
level=info ts=2026-06-29T22:19:44.074128417Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-06-29T22:19:44.074196863Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-06-29T22:19:44.074220333Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-06-29T22:19:44.074215614Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=1.775147961s
level=info ts=2026-06-29T22:19:44.074295313Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-06-29T22:19:44.074295674Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-06-29T22:19:44.074304958Z caller=loki.go:615 msg="Loki started" startup_time=88.209549ms
ts=2026-06-29T22:19:45.912691671Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:45.912717983Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-06-29T22:19:48.472735046Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:48.47277889Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T22:19:49.074497519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-06-29T22:19:53.834966978Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-06-29T22:19:53.834990656Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-06-29T22:19:54.075158881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:19:59.074392868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:04.074406068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:20:08.853592021Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=24.792204138s
level=warn ts=2026-06-29T22:20:09.075289822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:20:14.075013624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:20:14.075043214Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-06-29T22:20:14.075052068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:20:14.075088394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:19.075178793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:24.074680046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:29.075324113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:34.075273218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:39.075302676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:20:44.055919354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:20:44.075255404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:20:44.075295924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:20:44.075299711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:20:46.116346712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:20:46.130254303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD5DDB29C76E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:20:49.075030951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:54.074851777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:20:59.074957053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:20:59.987242314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:21:00.001032515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD6115EC3B4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:21:02.958858188Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAD61C63B9362, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T22:21:04.075266664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:09.074888303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:21:14.074939517Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:21:14.074974557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:21:14.074994188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:19.075320915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:24.075138928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:29.075360394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:21:31.263407668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:21:31.275974929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD685E0F34BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:21:34.075033872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:39.075064965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:21:44.055743258Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:21:44.074349592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:21:44.074383832Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:21:44.074394347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:49.074649892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:54.074943827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:21:59.075188791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:04.077230707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:09.075183036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:22:14.07439082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:22:14.074431364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:22:14.074463749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:22:16.609506173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:22:16.622874725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD72ECF166EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:22:19.074670389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:24.074699063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:29.075383295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:34.074364283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:39.074980203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:22:44.05656849Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T22:22:44.074454982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:22:44.074509014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:22:44.074542986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:22:49.075255971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:54.074571801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:22:59.075363657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:23:03.382238369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:23:03.395372192Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD7DD0CD7CD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:23:04.074845226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:09.075205239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:23:14.074672261Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:23:14.074675627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:23:14.07470556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:23:19.075429444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:24.074933716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:29.075130895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:34.075329671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:39.074769977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:23:44.05606504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:23:44.075049193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:23:44.075078998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:23:44.075083696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:23:44.313806937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:23:44.326949212Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD8758831B31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:23:49.074384015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:54.074361864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:23:59.075376304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:04.075123311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:09.075121749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:24:14.075128235Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:24:14.075157806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:24:14.075159283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:19.074847299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:24.075346827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:29.074398562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:34.07438767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:39.074540722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:24:39.588594499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:24:39.60156481Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAD943722C1F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:24:44.056230088Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:24:44.074221754Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:24:44.07435232Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-06-29T22:24:44.074360894Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:24:44.074365413Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:24:44.074379018Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.784µs success=true
level=info ts=2026-06-29T22:24:44.074392429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:24:44.074440157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:49.074416447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:24:53.735510215Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAD9781982B15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T22:24:54.07485637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:24:59.074958409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:04.075347209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:09.074730491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:25:14.074554823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:25:14.074591517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:25:14.07458487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:19.074857936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:24.075381252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:29.074756492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:25:33.026386805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:25:33.038313329Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADA0A8377EAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:25:34.075165298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:39.075387069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:25:44.055597384Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:25:44.075056956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:25:44.075088822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:25:44.075108484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:49.074717467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:54.074975397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:25:59.075346972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:04.074803239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:09.074617344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:14.074369279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:26:14.074382165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:26:14.074407363Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:26:19.074655019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:26:23.003163205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:26:23.014059087Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADAC4B002B29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:26:24.074398859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:29.074983092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:34.074873563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:39.074413986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:26:44.05594755Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:26:44.074999485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:26:44.075039905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:26:44.075072063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:49.074496876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:54.07524309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:26:59.074697641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:04.075094218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:06.787262922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:27:06.802280214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADB67CFC19F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:27:06.802303442Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:27:06.802328582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:27:06.81278355Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADB67D9CFD5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:27:09.074386306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:14.074881036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:14.074894212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:27:14.074929834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:27:19.075049522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:21.847834602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:27:21.859785074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADB9FE7B9D52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:27:24.07527366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:29.075092399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:34.074337864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:39.074576041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:44.055880274Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:27:44.074602354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:27:44.074618029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:44.074656784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:27:49.074561392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:27:54.075170277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:27:58.68340144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:27:58.696593306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADC292215DFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:27:59.07528345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:04.074464699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:09.074607861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:28:14.07494724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:28:14.074955727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:28:14.074982189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:28:19.074898673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:24.075179704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:29.075220283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:34.074910822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:39.074574515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:28:42.618594218Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDADCCCC17CB63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T22:28:44.055679151Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T22:28:44.075152889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:28:44.075156225Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:28:44.075185915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:28:45.843027002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:28:45.854436612Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADCD8CF4BAA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:28:49.074546753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:54.074415219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:28:59.075210583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:04.074978579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:09.075199391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:29:14.075153906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:29:14.075184442Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:29:14.075170373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:29:14.083386555Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-06-29T22:29:14.087147851Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012599691s
level=warn ts=2026-06-29T22:29:19.075263887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:24.074513479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:29.074710088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:34.074556477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:39.075153885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:29:41.855020437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:29:41.868385649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADDA97A3EF74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:29:44.055600317Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:29:44.074849541Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:29:44.074874855Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:29:44.074886745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:29:44.07492194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:29:44.075012583Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-06-29T22:29:44.075022599Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:29:44.075041007Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.658µs success=true
level=warn ts=2026-06-29T22:29:49.074979585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:54.074660683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:29:59.075198761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:04.074966312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:09.07505562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:14.074969669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:30:14.074990272Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:30:14.075019247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:30:19.074722912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:24.074558602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:29.074953602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:34.074380935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:30:34.954900552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:30:34.968218841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADE6F4A42276, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:30:39.07505131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:30:44.05563931Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:30:44.074858326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:30:44.074889591Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:30:44.074914763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:49.075066148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:54.074466618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:30:59.0752018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:04.074758878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:09.074906521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:31:14.075146152Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:31:14.075178256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:31:14.075172107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:19.074757038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:24.074355177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:29.074730069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:31:32.090518477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:31:32.104856403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDADF4423FC59A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:31:34.075102686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:39.074797999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:31:44.05634097Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:31:44.074996906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:31:44.075024396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:31:44.075046751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:49.075229527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:54.075022091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:31:59.074793554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:04.074900704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:09.07515886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:32:14.074853026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:32:14.074884236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:32:14.074904793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:19.074949369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:24.074818403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:29.074721602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:32:30.739244474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:32:30.752571196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE01E9EC8167, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:32:34.07501724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:39.074589743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:32:43.573249537Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE04E6187797, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T22:32:44.055579542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:32:44.074741977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:32:44.074773871Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:32:44.074829302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:49.075380496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:54.074419834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:32:59.074869372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:04.074621088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:09.074858169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:33:14.074520654Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:33:14.07456162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:33:14.074574874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:19.075062892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:33:22.979751408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:33:22.991718222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE0E139F3C39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:33:24.074538593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:29.075293178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:34.074511335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:39.074748607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:33:44.055847258Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:33:44.074335265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:33:44.074345145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:33:44.074370276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:33:49.074363099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:54.074597832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:33:59.075216855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:04.074760245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:06.982110757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:34:06.994502622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE18526640BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:34:06.994526782Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:34:06.99455346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:34:07.004462383Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE1852FF5BC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:34:09.075429871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:14.074537252Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:34:14.074568159Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:34:14.074570522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:14.083766696Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-06-29T22:34:14.084966913Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009793524s
level=warn ts=2026-06-29T22:34:19.075232957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:21.197232696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:34:21.213804674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE1BA1EEE927, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:34:24.074544565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:29.07535955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:34.075358748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:39.074554078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:44.056464735Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:34:44.074206218Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:34:44.07435288Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-06-29T22:34:44.074363472Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:34:44.074377165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:34:44.074386429Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.758µs success=true
level=info ts=2026-06-29T22:34:44.074401227Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:34:44.074412479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:49.075084307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:34:54.074398584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:34:57.166040385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:34:57.17896756Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE2401A0936B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:34:59.075317948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:04.075009889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:09.074580276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:35:14.074863879Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:35:14.07487248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:35:14.07490409Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:35:19.07485994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:24.075210909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:29.074992912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:34.074791949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:39.075120595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:35:39.388494064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:35:39.401885589Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE2DD64F6594, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:35:44.055848058Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:35:44.074709206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:35:44.074747604Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:35:44.074787124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:49.075269487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:54.075137896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:35:59.074748172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:04.074925618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:09.075349175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:36:14.074545008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:36:14.074563254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:36:14.07458573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:36:19.074992989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:36:23.574510046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:36:23.588251012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE382005BE2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:36:24.075172239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:29.075032435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:34.075289141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:39.075209448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:36:42.654717704Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE3C907A60B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T22:36:44.0563391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:36:44.07454618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:36:44.074580026Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:36:44.074596945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:49.074733437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:54.074370294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:36:59.075211482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:04.074535086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:09.074458772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:37:14.074339108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:37:14.074373377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:37:14.074390236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:19.075134187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:37:21.632751098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:37:21.644674806Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE45A472F54A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:37:24.074400209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:29.074609548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:34.07465196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:39.075049412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:37:44.055850354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:37:44.074304321Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:37:44.074344712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:37:44.074426881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:49.07466399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:54.075278418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:37:59.074348403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:38:01.918860511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:38:01.930001253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE4F05A54D13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:38:04.075224499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:09.074548927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:38:14.074838275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:38:14.074870197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:38:14.074874302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:19.075125869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:24.075214069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:29.075014902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:34.074893777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:39.075067685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:38:44.055992661Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:38:44.074504074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:38:44.074537473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:38:44.074544512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:49.074517301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:54.075258238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:38:59.074460901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:39:00.13721988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:39:00.14945515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE5C93CC3AE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:39:04.074586669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:09.075328423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:39:14.074481324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:39:14.074517902Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:39:14.074568071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:39:14.07903782Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-06-29T22:39:14.080797457Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006271598s
level=warn ts=2026-06-29T22:39:19.075112265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:24.075139948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:29.074612134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:34.074801612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:39.07527085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:39:44.055657094Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:39:44.07449507Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:39:44.074515914Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:39:44.074551244Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:39:44.074614947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:39:44.074647633Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-06-29T22:39:44.074656722Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:39:44.074674712Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.388µs success=true
level=info ts=2026-06-29T22:39:45.047068251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:39:45.060464944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE6708B28E5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:39:49.074943875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:54.075036568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:39:59.074890052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:04.075230843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:09.074674216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:40:14.07497248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:40:14.074989493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:40:14.07500808Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:40:19.074841861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:24.075253764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:29.074391753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:34.074947428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:40:35.990212346Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAE72E45C2A73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T22:40:37.647478022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:40:37.65835498Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE7347C619C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:40:37.658377045Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:40:37.658398556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:40:37.668670272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE7348637BB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:40:39.075395707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:40:44.055840121Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:40:44.075229477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:40:44.075260898Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:40:44.075278322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:49.074679784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:40:54.074876583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:40:57.656261011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:40:57.668097625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE77F072E507, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:40:59.075224602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:04.074581359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:09.075084533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:14.074392145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:41:14.074430043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:41:14.074464237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:41:19.07450568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:24.075012313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:29.075289374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:41:31.644978309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:41:31.658793217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE7FDA720F25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:41:34.075289347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:39.074893388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:41:44.056504282Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T22:41:44.074467073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:41:44.074488617Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:41:44.074512824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:41:49.0750809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:54.075179739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:41:59.07500957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:04.074654262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:09.075141245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:42:14.074603204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:42:14.074650066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:42:14.074670736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:42:14.459872365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:42:14.473200447Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE89D2644F01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:42:19.074476565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:24.075226576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:29.075387338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:34.074968752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:39.074508905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:42:44.056050393Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:42:44.074919131Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:42:44.074953606Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:42:44.075001593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:49.075318503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:42:54.07489728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:42:56.54645356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.7 MB" avg_comp="3.7 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:42:56.560189836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE939EFA004C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:42:59.074762229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:04.075198905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:09.074712682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:43:14.0753344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:43:14.075374766Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:43:14.07539898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:19.075043098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:24.074331402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:29.074899165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:34.075163569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:39.07733113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:43:44.055645139Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:43:44.074437155Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:43:44.074478611Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:43:44.074473077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:43:44.651673216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:43:44.662941077Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAE9ED21F60BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:43:49.075391243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:54.074758407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:43:59.07498396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:04.074424375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:09.07491063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:14.075219275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:44:14.07523282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:44:14.07525736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:44:14.08061949Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-06-29T22:44:14.082503654Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007699258s
level=warn ts=2026-06-29T22:44:19.075007868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:24.074656259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:44:26.597455162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:44:26.611133248Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEA8966EA5F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:44:29.074915858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:34.074903746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:39.075047653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:44:44.055907723Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:44:44.074959908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:44:44.074976354Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-29T22:44:44.074973434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:44:44.074993222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:44:44.075112964Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-06-29T22:44:44.075120909Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:44:44.075137883Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.426µs success=true
level=warn ts=2026-06-29T22:44:49.074523158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:54.074860849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:44:59.07492903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:04.074678815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:45:08.476883994Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAEB255D391DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T22:45:09.075396625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:45:14.074360213Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:45:14.074373012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:45:14.074399751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:45:19.074612559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:45:22.897048331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:45:22.911545111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEB5B231CF9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:45:24.074671529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:29.074801203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:34.074995301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:39.075397321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:45:44.055621684Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:45:44.074806418Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:45:44.074806991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:45:44.074840231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:45:49.074985509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:54.075320373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:45:59.074979547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:46:03.531601058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:46:03.544001544Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEBF2813C50E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:46:04.074455285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:09.074949641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:46:14.074737154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:46:14.074766036Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:46:14.074758152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:19.0752544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:24.074926449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:29.074986286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:34.075347483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:39.074691781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:46:44.056305007Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:46:44.074556986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:46:44.074587519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:46:44.074587242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:49.074939135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:46:54.074805162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:46:58.248764963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:46:58.259562849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAECBE5612E6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:46:58.259595855Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:46:58.259645739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:46:58.269416371Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAECBE5F95A40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:46:59.074344963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:04.074655994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:09.075253079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:47:14.07440306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:47:14.074443162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:47:14.074471958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:47:18.040046277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:47:18.051972792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAED081195899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:47:19.075226274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:24.074965597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:29.075067571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:34.075309998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:39.075073687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:47:44.055803664Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:47:44.075310172Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:47:44.075342858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:47:44.075356157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:49.075150244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:47:49.973838356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:47:49.98538434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAED7F079D6B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:47:54.075085164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:47:59.074727899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:04.074344727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:09.074452039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:14.074360442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:48:14.074379835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:48:14.074403758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:48:19.074739902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:48:21.059281029Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAEDF2CA2C936, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T22:48:24.075407006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:29.075221668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:48:31.506596261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:48:31.521853841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEE19C3DCFCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:48:34.074381893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:39.074998344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:48:44.055882363Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:48:44.075272161Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:48:44.075304975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:48:44.075309431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:49.074853662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:54.074853702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:48:59.074962173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:04.075228232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:09.074989096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:49:14.075308988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:49:14.075347408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:49:14.075386726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:49:14.081474197Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-06-29T22:49:14.08333731Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008077859s
level=warn ts=2026-06-29T22:49:19.075202157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:24.074647506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:29.074644382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:49:29.176900622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:49:29.189167307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEEF097C4715, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:49:34.074745863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:39.075322431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:49:44.056528492Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:49:44.074162742Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:49:44.07431396Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-06-29T22:49:44.074323653Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:49:44.074341883Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.773µs success=true
level=info ts=2026-06-29T22:49:44.074398267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:49:44.074442478Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:49:44.074464775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:49.074946466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:54.075323236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:49:59.074680554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:04.074967064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:09.074567793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:14.075103731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:50:14.075135795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:50:14.075158825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:16.989507356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" idle=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:50:16.995909622Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEFA2AFBB74E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:50:17.290436263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="132 B" avg_comp="132 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:50:17.293838112Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEFA3CBEBAE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:50:19.048158682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:50:19.056929942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEFAA5D4C9C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:50:19.074656712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:24.07474693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:29.074715659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:30.768524686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:50:30.776698643Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEFD606207E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:50:31.08266003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:50:31.085831601Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAEFD72CFA4B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T22:50:34.075173942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:39.075449989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:44.055929113Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:50:44.074662605Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:50:44.074690886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:50:44.07468758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:50:49.0751164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:52.875964033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="385 kB" avg_uncomp="385 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:50:52.879595681Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0285D1BA90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:50:53.17639628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="131 kB" avg_uncomp="131 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:50:53.180243149Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0297BD5CB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:50:53.414752202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:50:53.418296667Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02A5ED9AB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:50:53.476585207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:50:53.479979981Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02A99B8DB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:50:53.776821871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:50:53.780199338Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02BB813E75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:50:54.07434232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:50:54.076419321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:50:54.079851024Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02CD5C7553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:50:54.376914478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:50:54.380461837Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02DF468267, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:50:54.677014172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:50:54.680374672Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF02F128C94F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:50:54.976001947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:50:54.979162698Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0302F88E33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:50:55.27629503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:50:55.279808588Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0314E1C800, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:50:55.576686048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="146 B" avg_comp="146 B" total_uncomp="23 kB" avg_uncomp="23 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:50:55.580111317Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0326C86606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:50:55.875992833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:50:55.879595573Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0338A2F3FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:50:56.176515307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:50:56.179872631Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF034A8808C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:50:56.476801521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:50:56.480087798Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF035C6EA642, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T22:50:59.074920184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:01.139744505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:51:01.143262026Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF04725F848E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T22:51:04.075160383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:06.099644326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:51:06.102950978Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0599FEB159, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:51:07.130673169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:51:07.133961028Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF05D7721C5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:51:07.648272969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:51:07.651385441Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF05F64AB08A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:51:07.996277507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:51:07.999788131Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF060B0CC45B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:51:08.138115796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:51:08.141601582Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF061380C807, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:51:08.349409899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:51:08.352779181Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0620178805, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:51:08.810975493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:51:08.818749911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF063BDD8823, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:51:08.904256157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:51:08.910164946Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF064151D3EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:51:09.074882583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:09.130980246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:51:09.134207743Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF064EAC9A1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:51:09.403210962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:51:09.406498027Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF065EE7BE28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:51:09.891782055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:51:09.895001403Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF067C043804, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:51:12.771233852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:51:12.774720738Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0727A99A66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:51:13.184238214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:51:13.187620934Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0740462E15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:51:14.074659407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:51:14.07469165Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:51:14.074677853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:14.130001368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:51:14.133280882Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0778A3A219, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T22:51:19.075385992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:51:24.074863562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:51:29.074413921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:31.061420015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:51:31.064734898Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0B69D4E9F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:51:32.770601573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:51:32.773967219Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0BCFB6A596, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:51:34.074654125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:34.427881864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:51:34.431308539Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0C327D969B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:51:34.526681157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:51:34.530040688Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0C38621251, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:51:35.729043045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:51:35.732523408Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0C800D8F1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:51:39.074359659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:39.11150453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:51:39.114815265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0D49A7CCDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:51:39.17597852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:51:39.179211591Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0D4D7F2779, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:51:39.467662963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:51:39.470788683Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0D5EE052BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:51:39.757207607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:51:39.760330576Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0D7022D264, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:51:39.910813121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:51:39.914032793Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0D794AC1FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:51:42.081516788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:51:42.085090771Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0DFAB25759, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:51:42.311678335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:51:42.314758384Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0E08648955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:51:42.879365839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:51:42.882702948Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0E2A3D07C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:51:42.954900055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:51:42.958023907Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0E2EBBC0B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:51:44.055904546Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:51:44.074457547Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:51:44.074483053Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:51:44.074474664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:47.184152105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:51:47.187523446Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF0F2AD37823, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T22:51:49.0754102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:51:52.806547301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:51:52.81447307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF107A38D5A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:51:54.074795462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:51:59.075029296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:52:04.075048126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:52:09.075422318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:12.639266483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:52:12.646826534Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF151852263D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:52:14.074637587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:52:14.074648119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:14.074671066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:52:15.833858971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:52:15.837231558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF15D67BB41E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:52:16.941023947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:52:16.944356647Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF161879B8C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T22:52:19.074955927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:21.654587781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:52:21.657792572Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF17316ACD44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:52:24.075276343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:25.076749318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:52:25.079950794Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF17FD64465C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:52:25.557853785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:52:25.561235119Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF181A138D61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:52:29.045185021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:52:29.048494803Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF18E9EEBEAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T22:52:29.074825763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:29.272560067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:52:29.276046011Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF18F780B2FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:52:30.960366111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:52:30.963732743Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF195C175F4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:52:31.017760927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:52:31.020925137Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF195F80EFA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:52:32.58790349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:52:32.591645761Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF19BD1E4B97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:52:33.516810265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:52:33.520054039Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF19F47755E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:52:34.074765594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:34.908208565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:52:34.911801856Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF1A4769A14D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:52:35.517772721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:52:35.521065728Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF1A6BBBA2A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:52:39.074584946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:42.557398141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:52:42.560797147Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF1C0F55013E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:52:43.099680077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:52:43.107643276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF1C2FEC4570, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:52:44.055592943Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:52:44.074370751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:52:44.07436475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:52:44.074400847Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-06-29T22:52:44.302035678Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF1C771F964E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T22:52:49.074614668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:52:54.075329624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:52:59.075222183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:53:04.074926768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:04.219212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:53:04.226656834Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF211AB5BF50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:53:05.004436422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:53:05.007790411Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2149467B57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:53:08.100834515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:53:08.104218426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2201D61DE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:53:09.075212464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:10.318202281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:53:10.321620861Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF22860159D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:53:10.977421518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:53:10.980873676Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF22AD4C621D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:53:11.405562144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:53:11.408907394Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF22C6D04A0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:53:14.022056287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:53:14.02543718Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2362C5E102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:53:14.074875926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:14.074897176Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:53:14.074922132Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:53:17.632740571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:53:17.636152885Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2439FC4559, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:53:19.033693954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:53:19.037217995Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF248D7DAAFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:53:19.075374577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:23.128747113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:53:23.132440077Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF258195E799, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T22:53:24.0750132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:25.239332777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:53:25.242957071Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF25FF62AF9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:53:25.691533089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:53:25.695271547Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF261A58A143, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:53:27.906975569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:53:27.910581943Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF269E637615, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:53:29.074684803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:30.884605843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:53:30.893414557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF27502D43EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:53:30.893436058Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:53:30.893460844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:53:30.904412602Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2750D338F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:53:33.678225972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:53:33.683903238Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF27F68125B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T22:53:34.075354929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:34.567281446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:53:34.573607361Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF282B891640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T22:53:39.074428254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:40.115018969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:53:40.121356356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2976349547, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:53:44.055635767Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:53:44.074807329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:53:44.074845645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:53:44.074877117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:44.384262925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:53:44.392185923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2A74C614F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:53:46.751246539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:53:46.757555481Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2B01C1B897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:53:49.070076142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T22:53:49.074841867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:53:49.075617796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2B8BEBFF48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:53:52.18357808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:53:52.189773263Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2C458A586E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:53:53.416157471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:53:53.419779026Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2C8ED91295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:53:54.075059033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:53:59.0743579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:53:59.720347442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:53:59.72395541Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2E069C60A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:54:04.074942452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:05.870302579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:54:05.8737569Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2F752C5249, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:54:06.752672442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:54:06.756009306Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2FA9C12CFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:54:07.59759433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:54:07.601035928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF2FDC2015E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T22:54:09.074935978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:09.990891266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:54:09.994476326Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF306AC97BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:54:10.160029781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:54:10.163392047Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3074DB349F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:54:10.817263477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:54:10.820691637Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF309C07D47C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:54:10.976226133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:54:10.979778463Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF30A5818972, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:54:11.21634974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:54:11.221814382Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF30B3F0DE29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:54:13.186829544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:54:13.197990918Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3129BA8A28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:54:14.075117107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:54:14.075149782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:54:14.075143466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:14.082150245Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-06-29T22:54:14.083741711Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009256631s
level=info ts=2026-06-29T22:54:16.775678755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:54:16.781497363Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF31FF526935, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:54:19.075091364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:19.154387113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:54:19.157717733Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF328CF3992F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:54:21.021570433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:54:21.024893036Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF32FC3F29C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T22:54:24.074360683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:54:29.075202565Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:31.472192437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:54:31.47747141Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF356B42BEB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T22:54:34.074786772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:37.161053035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:54:37.164671863Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF36BE3FEB35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:54:39.001920034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:54:39.005322648Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF372BF783E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:54:39.074929035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:43.062969539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:54:43.066591525Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF381E07F0D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:54:44.055653636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:54:44.074728366Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:54:44.074774619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:54:44.074791284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:44.074803722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:54:44.074867892Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-06-29T22:54:44.07487614Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:54:44.074893843Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.928µs success=true
level=warn ts=2026-06-29T22:54:49.074962577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:51.608740855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:54:51.61206041Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3A1B6356B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:54:52.702952829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:54:52.706549704Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3A5C9D4237, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:54:53.876882339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:54:53.880240918Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3AA293A2AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T22:54:54.074563882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:54:56.457454317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:54:56.460958994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3B3C66881C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:54:56.880151191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:54:56.885798425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3B55B7D92C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:54:57.42020056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:54:57.426997972Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3B75FD3AC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:54:59.074531491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:00.541742605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:55:00.544968533Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3C2FD5671C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:55:03.21758559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:55:03.220923562Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3CCF55C269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:55:03.412780141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:55:03.416060762Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3CDAF6E20C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T22:55:04.075215493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:05.188075944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:55:05.195947004Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3D450D537C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:55:07.375252141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:55:07.381721614Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3DC7525DFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:55:09.074494981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:10.182072536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:55:10.185823298Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3E6E75D3A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:55:10.750663402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:55:10.753813935Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF3E90538730, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:55:14.074794471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:55:14.074807255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:14.074827472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:55:19.074656561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:19.992421091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:55:19.995969213Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF40B731A34C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T22:55:24.074366478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:28.581910607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:55:28.585550024Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF42B72C19D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T22:55:29.075267342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:31.052213466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:55:31.055593491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF434A669EFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:55:33.026285562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:55:33.029903811Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF43C013D45D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T22:55:34.074509894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:38.750006482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:55:38.753432685Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF45153BBEAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:55:39.075161899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:42.781647033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:55:42.785072388Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF460587DFB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:55:44.05565503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:55:44.074813873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:55:44.074831389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:44.074845257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:55:44.898832862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:55:44.902232743Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4683BACC6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:55:48.606374278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:55:48.609858883Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4760B7BC86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:55:49.074403737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:50.416968826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:55:50.420394413Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF47CCA07C75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:55:50.900295498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:55:50.906217608Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF47E9953A8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:55:53.477737584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:55:53.484207774Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF488341E7F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T22:55:54.074646658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:56.234550153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:55:56.237989862Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF49276499F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T22:55:59.074731265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:55:59.72844788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:55:59.731880478Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF49F7A5519E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:56:01.940272078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:56:01.943650667Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4A7B7A0351, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T22:56:04.074812549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:05.196193159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:56:05.204087957Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4B3DD0D97F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:56:05.213164847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:56:05.220739412Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4B3ECF31A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:56:07.351193056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:56:07.354607015Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4BBDFEAADF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:56:09.074383861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:14.074410989Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:56:14.074418857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:14.074444607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:56:14.634679062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:56:14.637888186Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4D701D2168, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:56:15.261104021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:56:15.264166361Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4D9571E78B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:56:15.26418311Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:56:15.264200738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:56:15.266980541Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4D959F6BDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:56:16.352004161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:56:16.355147955Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4DD679BB29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:56:16.355165648Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:56:16.355183132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:56:16.357878385Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4DD6A444EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:56:17.926040864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:56:17.929210667Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4E344B0FC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T22:56:19.075374042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:23.064364229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:56:23.067570599Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4F66907A46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:56:24.075208621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:25.338115457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:56:25.341489381Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF4FEE18641E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:56:28.424591904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:56:28.427950477Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF50A61050F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T22:56:29.074923691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:33.709257679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:56:33.712577733Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF51E10E9A8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:56:34.074745969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:34.837886479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:56:34.841099031Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF522451C929, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:56:35.615255839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:56:35.618513144Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5252A7FAEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T22:56:39.075394978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:42.79567576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:56:42.799068277Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF53FEA76DBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:56:44.055883233Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:56:44.074319607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:56:44.074350771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:56:44.074342124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:46.509842914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:56:46.513160688Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF54DC075350, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:56:46.534210842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:56:46.539581951Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF54DD9A2011, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:56:48.323959966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:56:48.336643644Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5548B6F761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:56:49.07451723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:50.182329348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:56:50.188739166Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF55B718244A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:56:51.975417075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:56:51.979071599Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5621D0B885, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:56:52.245746964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:56:52.249032717Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5631E9797D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T22:56:54.074859656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:56:57.327223436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:56:57.330685224Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5760CBA0F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T22:56:57.330705782Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:56:57.330726186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:56:57.333680647Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5760FBD4EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:56:58.896673818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:56:58.900196411Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF57BE59C521, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:56:59.075320771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:01.933090196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:57:01.936658351Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF58735534DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:57:02.773473512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:57:02.776760555Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF58A569BA5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:57:04.074663681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T22:57:05.498396286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF5947A5144F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T22:57:05.987892335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:57:05.991297094Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF596503C35E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:57:07.723859423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:57:07.72724509Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF59CC7C3724, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:57:07.727264706Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:07.727282805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:57:07.729965614Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF59CCA7AF0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:57:07.888531608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:57:07.89204214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF59D64D7BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T22:57:09.075269359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:09.376203155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:57:09.379665211Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5A2EF793E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T22:57:14.07437592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:14.074430084Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:57:14.074458479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:57:16.877893536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:57:16.881151394Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5BEE19947C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:57:16.881167762Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:16.881186136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:57:16.88411553Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5BEE467B22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:57:18.564588036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:57:18.568037933Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5C52A368E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:57:18.568062348Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:18.568090338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:57:18.570928702Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5C52D2332A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T22:57:19.075119046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:21.962250858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:57:21.965538605Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5D1D27190E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:57:24.07504322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:24.320038014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:57:24.32339457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5DA9B067C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-29T22:57:24.32341089Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:24.323429371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:57:24.326096131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5DA9DCD213, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:57:29.074391103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:30.333197466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:57:30.341214381Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5F1060344F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:57:31.506644579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:57:31.514398383Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5F564E879C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T22:57:34.074811407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:34.079103499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:57:34.082667255Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5FEF624C4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:57:34.082686612Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:34.08270519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:57:34.085691928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF5FEF9218A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:57:34.520899439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:57:34.52419591Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6009B4C58A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:57:35.526329907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:57:35.529701143Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6045A1C6DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:57:36.170673726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:57:36.174110856Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF606C0D23F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:57:36.174128707Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:36.174146354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:57:36.176971471Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF606C39D61E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:57:36.941085495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:57:36.94432292Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6099F4EEE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:57:39.074841042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:39.109009111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:57:39.114751147Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF611B532BF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:57:39.861887939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:57:39.868529433Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF614840DC11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:57:39.868549759Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:39.868571609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:57:39.871424961Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF61486EF379, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:57:40.492756098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:57:40.496677135Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF616DB128B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:57:40.49669721Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:40.496717151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:57:40.499530129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF616DDF251A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:57:43.064368422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:57:43.067836929Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6206F19330, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:57:43.067863633Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:43.06789016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:57:43.070701176Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF62071F8B54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:57:44.055601755Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:57:44.074329852Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:57:44.074339606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:44.074360577Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:57:45.797612548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:57:45.801202575Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF62A9DD3B6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:57:47.974909554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:57:47.978351457Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF632BA2FE90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T22:57:49.075288164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:51.013055024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:57:51.016545956Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF63E0B97305, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:57:51.293482941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:57:51.297037046Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF63F16F9556, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:57:51.664034484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:57:51.667293677Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6407847898, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:57:51.667310681Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:51.667333181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:57:51.670155539Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6407B185C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:57:52.417618809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:57:52.420807501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF64346DE497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:57:52.485784333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:57:52.489061987Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF64387F3AEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:57:52.489079643Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:52.489100597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:57:52.492016921Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6438AE227B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:57:53.224466737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:57:53.227797497Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6464870C2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:57:53.227814839Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T22:57:53.227834987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:57:53.230669719Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6464B3EF1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:57:54.074923273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:57:56.895343977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:57:56.898682511Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF653F545EE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:57:57.873594778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:57:57.877033961Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6579A4350C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:57:58.859735151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:57:58.863211096Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF65B46B5C8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T22:57:59.074742803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:00.651614374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:58:00.655291073Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF661F3AA82D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:58:02.366497322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:58:02.369804742Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF66856FA799, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:58:04.074441587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:04.468496635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:58:04.471955901Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6702BAE400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T22:58:06.072716853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:58:06.076098459Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6762583D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T22:58:09.074816791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:10.058664734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:58:10.062052519Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF684FEAFF71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:58:11.731374001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:58:11.734856954Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF68B3A11412, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:58:14.07499307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:58:14.075000427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:14.075041466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:58:14.709426206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:58:14.712820653Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF696522036A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T22:58:16.284885867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:58:16.2929796Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF69C351308F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T22:58:16.934653894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:58:16.941784396Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF69E9FE63A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T22:58:17.381203694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:58:17.384785199Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6A0464276F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T22:58:19.075252339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:21.76877166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:58:21.774572236Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6B0A0C892F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T22:58:23.968330074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:58:23.97467205Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6B8D2F8975, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T22:58:24.075344924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:58:29.074370901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:32.265340241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:58:32.268599508Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6D7B8B4262, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T22:58:32.525212923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:58:32.528601034Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6D8B09C5B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:58:34.074957576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:34.119046486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:58:34.122298717Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6DEA08B642, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:58:36.462593968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:58:36.46588137Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6E75B85987, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T22:58:39.075119483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:40.386241042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:58:40.389940967Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6F5F9D036F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T22:58:41.570017995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:58:41.573426091Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6FA627DC97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:58:43.018148696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:58:43.021953502Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF6FFC7DFFBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T22:58:44.056422132Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:58:44.075066838Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T22:58:44.07509818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:44.075121855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:58:47.139315025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:58:47.142675108Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF70F21B5F4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T22:58:49.074831453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:51.098675979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:58:51.102168944Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF71DE1C304D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T22:58:54.074826268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:58:56.446684945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:58:56.450017743Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF731CDE4F89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T22:58:58.367126692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:58:58.370740102Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF738F59D440, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T22:58:59.074886194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:01.60783285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:59:01.611274171Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF74507FB351, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:59:03.93592184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:59:03.93931537Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF74DB436BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T22:59:04.075063182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:06.888437003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T22:59:06.891928205Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF758B3FB4E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T22:59:07.664383579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T22:59:07.672389865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF75B9C5A454, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T22:59:09.075035531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T22:59:14.074471316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:14.074495482Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:59:14.074522023Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T22:59:14.080678187Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-06-29T22:59:14.082303245Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00729443s
level=info ts=2026-06-29T22:59:17.064789616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T22:59:17.072163999Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF77EA0A0B92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T22:59:17.386134223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T22:59:17.3894661Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF77FCF41D2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T22:59:18.103995726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T22:59:18.109508377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7827DF55CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T22:59:19.075095637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:19.259118371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T22:59:19.26499287Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF786CBDCE6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T22:59:24.074509729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:25.307452788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T22:59:25.310885913Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF79D51BD627, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T22:59:26.142032106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T22:59:26.145438836Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7A06DA256F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T22:59:29.074972098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:32.117654605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T22:59:32.121154014Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7B6B07EBC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T22:59:34.07468137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:39.014581856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T22:59:39.018264109Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7D061F5028, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T22:59:39.032307697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T22:59:39.035368476Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7D07279FA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T22:59:39.074610537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:40.656219124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T22:59:40.659896362Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7D67FAF1DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T22:59:44.055791359Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T22:59:44.074506665Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T22:59:44.074564425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T22:59:44.074596167Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T22:59:44.074593041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:44.074680533Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-06-29T22:59:44.07468991Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T22:59:44.07470938Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.707µs success=true
level=info ts=2026-06-29T22:59:45.32254342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T22:59:45.326121486Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7E7E1B1180, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T22:59:49.074545717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:49.226956555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T22:59:49.230512859Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7F66D317EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T22:59:50.791937647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T22:59:50.795251687Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF7FC4174BD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T22:59:53.812142306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T22:59:53.815639176Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF80781EB73A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T22:59:54.074400288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T22:59:58.984688815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T22:59:58.988184892Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF81AC6D9D8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T22:59:59.074666563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:03.444402585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:00:03.448087058Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF82B6412560, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:00:04.074404815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:05.598829051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:00:05.602389744Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8336A9C9A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:00:06.150944303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:00:06.156612428Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8357B2F5EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:00:06.156645674Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:00:06.156670702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:00:06.16597899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8358422D5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:00:07.190687857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:00:07.201540589Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8395FB8664, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:00:07.201562754Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:00:07.201588843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:00:07.211311363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF839693034A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:00:09.075076137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:10.558440768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:00:10.564388073Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF845E6D43E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:00:14.074655659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:00:14.074668392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:14.074696583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:00:16.7626789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:00:16.76623179Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF85D0154F04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:00:16.844296919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:00:16.850021902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF85D5149DCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:00:17.792684373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:00:17.798809794Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF860DA20AD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:00:19.07520294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:22.591016859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:00:22.599001145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF872BBFC6BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:00:23.474499681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:00:23.481032469Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF87604FDEA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:00:23.720574898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:00:23.723897892Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF876ECAFB81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:00:24.074814705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:25.314208201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:00:25.317691824Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF87CDC9C700, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:00:28.466574891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:00:28.470061284Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8889AF086E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:00:29.074535385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:00:34.075083053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:34.391281368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:00:34.394826633Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF89EAD41C4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:00:34.419665534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:00:34.422907124Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF89EC81165C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:00:37.849565202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:00:37.852937788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8AB8F30519, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:00:38.290193786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:00:38.293793717Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8AD3390830, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:00:38.766946354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:00:38.772598657Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8AEFC416DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:00:39.075015975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:39.684332949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:00:39.692010895Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8B268ED146, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:00:40.648350134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:00:40.65170428Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8B5FC5BE4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:00:44.05603339Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:00:44.074317667Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:00:44.074349541Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:00:44.07436452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:46.762666122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:00:46.766125186Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8CCC379B40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:00:47.338913036Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAF8CEE5F579D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-29T23:00:49.075307019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:53.857923293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:00:53.866457506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8E736EF630, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:00:54.074745239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:00:58.165840377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:00:58.173583189Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF8F7428A9D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:00:59.075074375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:00.963603071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:01:00.96718113Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF901AA94BEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:01:01.251098305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:01:01.254358812Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF902BCA0066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:01:02.915701335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:01:02.919076243Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF908F03D49E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:01:04.075146795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:06.05861603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:01:06.061985989Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF914A56D867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:01:09.075392052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:14.074960831Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:01:14.074993394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:01:14.075028924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:18.615150578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:01:18.621619979Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9436DB13C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:01:19.075137856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:19.731067916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:01:19.734510507Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF947949BD95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:01:22.291243167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:01:22.294774109Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9511E37976, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:01:22.78516449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:01:22.788547924Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF952F527F16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:01:22.871816983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:01:22.87517209Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF95347D2407, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:01:22.981477886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:01:22.984922913Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF953B071427, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:01:24.074391212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:25.224860511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:01:25.228301466Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF95C0BED429, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:01:25.682363158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:01:25.688458254Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF95DC2B7D2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:01:28.148402075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:01:28.155841443Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF966F3CB678, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:01:29.075285045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:31.085583587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:01:31.088973789Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF971E1012F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:01:34.075147363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:01:39.075345482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:40.203152084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:01:40.206685419Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF993D84CFC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:01:41.028000663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:01:41.031496739Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF996EAF6EA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:01:44.056569316Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:01:44.074400608Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:01:44.074439567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:01:44.074444345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:45.551242847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:01:45.554787039Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9A7C4C072A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:01:48.98243441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:01:48.985858267Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9B48CCFB87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:01:49.074918048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:52.523074852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:01:52.531879307Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9C1C294A7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:01:54.075149165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:01:55.953730002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:01:55.960738573Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9CE8897632, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:01:59.075272587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:02.425697965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:02:02.429306971Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9E6A176E6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:02:03.930415589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:02:03.933968928Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9EC3C5F2A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:02:04.074435498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:06.140863083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:02:06.144387008Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9F47882916, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:02:06.262803877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:02:06.266011027Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9F4EC8D0B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:02:08.918555199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:02:08.922057066Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAF9FED17C21B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:02:09.07536345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:14.075071302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:02:14.075104877Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:02:14.075100087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:14.506996134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:02:14.512646691Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA13A50A6BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:02:15.137228585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:02:15.144731285Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA15FFD9DE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:02:15.443676738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:02:15.446938218Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA172026C1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:02:16.831024183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:02:16.83432239Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA1C4B2EA56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:02:19.074419006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:21.917084205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:02:21.92005335Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA2F3D81EF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:02:21.955263986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:02:21.958310682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA2F61E2DD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:02:24.074360204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:27.196519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:02:27.199858741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA42E87D972, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:02:29.074514179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:31.118001926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:02:31.121436878Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA51847A483, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:02:34.074762268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:02:39.074638385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:40.063901635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:02:40.06717784Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA72D7D1F0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:02:42.126071022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:02:42.13388178Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA7A8AE75B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:02:44.056351656Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:02:44.074919114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:02:44.074948072Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:02:44.075009768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:47.645749446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:02:47.652419197Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA8F19B58CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:02:47.943559473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:02:47.946863934Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFA903278AD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:02:49.074598016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:53.795649139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:02:53.799221072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAA5FF9E335, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:02:53.799252394Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:02:53.799280728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:02:53.802096059Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAA60290A9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:02:54.074682453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:02:55.795597702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:02:55.798993679Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAAD72D6526, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:02:55.799009627Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:02:55.799027788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:02:55.801585232Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAAD757451F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:02:56.774898213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:02:56.778183838Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAB118BA478, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:02:57.941967515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:02:57.945159517Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAB571A5628, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:02:59.074757852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:00.93209915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:03:00.935352052Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAC09547768, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:03:02.462043849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:03:02.46553829Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAC6488447E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:03:02.485574376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:03:02.490950083Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAC660D7B13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:03:04.074589399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:06.336920075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:03:06.344389566Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAD4BBB6790, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:03:07.015112415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:03:07.018310649Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAD73E784D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:03:08.862582859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:03:08.865974739Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFADE206C928, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:03:09.074931052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:10.027502414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:03:10.030821278Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAE2775C741, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:03:11.601836857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:03:11.605241541Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAE854D74AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:03:13.719399696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:03:13.72281175Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAF03846C96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:03:14.074502554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:03:14.074533722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:03:14.074552945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:14.887585369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:03:14.890966867Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFAF49258B64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:03:19.074725405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:21.793036822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:03:21.796542519Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB0E4C09EBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:03:24.075323364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:28.562671955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:03:28.565896475Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB2783D0773, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:03:28.57493831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:03:28.577859759Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB278F4F3AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:03:28.577872373Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:03:28.577889406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:03:28.580458313Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB2791E75D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:03:29.074908613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:31.932430326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:03:31.940652537Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB341643B03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:03:34.074828049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:37.189063228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:03:37.197467979Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB47AB63A48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:03:39.074863379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:40.922361084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:03:40.925700051Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB558EF961C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:03:41.717827388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:03:41.721170706Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB58858BBBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:03:43.253672613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:03:43.256878976Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB5E3E215FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:03:43.351210744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:03:43.357246787Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB5E9DEF953, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:03:44.055741803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:03:44.075111566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:03:44.075116584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:44.075144661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:03:44.954899082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:03:44.960671047Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB649701730, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:03:44.960697368Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:03:44.960723947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:03:44.963431026Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB6499DFB2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:03:49.074566681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:50.249241335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:03:50.252424625Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB784DB5F2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:03:54.075044251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:03:54.480736097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:03:54.484235322Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB881173209, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:03:54.484257347Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:03:54.484281408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:03:54.486994947Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB881435DB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:03:56.222933787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:03:56.226513034Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB8E8EE67A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-29T23:03:56.226531588Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:03:56.226549179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:03:56.229314161Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB8E91D1C5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:03:59.074368237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:00.473574975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:04:00.476938409Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFB9E6481D75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:04:03.361917015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:04:03.36524817Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBA9271B975, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:04:04.075114308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:05.769927773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:04:05.773155013Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB21F6C9DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:04:05.773175636Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:05.773196464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:04:05.775957483Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB2223A074, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:04:06.372676624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:04:06.37589698Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB45E34C17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:04:06.375916601Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:06.375935836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:04:06.378742617Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB4610A713, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:04:06.535091262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:04:06.538191598Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB4F90B0AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:04:07.440557415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:04:07.443996922Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB858BC4BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:04:07.444020245Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:07.444044156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:04:07.446836917Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBB85B96F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:04:08.162029038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:04:08.165196492Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBBB08B07A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:04:09.074670535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:10.094092184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:04:10.097383023Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBC23B48791, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:04:10.097408309Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:10.097433212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:04:10.100096556Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBC23E04FBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:04:12.062594997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:04:12.065890196Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBC9909F0B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T23:04:12.065907958Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:12.065926578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:04:12.068603701Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBC99357775, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:04:14.075137429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:14.075158118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:04:14.075185427Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:04:14.082189357Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-06-29T23:04:14.083896516Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009054321s
level=info ts=2026-06-29T23:04:14.917276777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:04:14.92073925Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBD43329EA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:04:19.075355158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:19.654052852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:04:19.657438581Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBE5D8831AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:04:20.345683383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:04:20.353385898Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBE8702DF80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:04:20.985026984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:04:20.991107217Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBEAD04EA5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:04:21.661175391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:04:21.664492699Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBED528DADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:04:21.664510652Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:21.664529815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:04:21.667259108Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBED555677E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:04:23.378440192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:04:23.381951099Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBF3B84CDD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:04:24.074932282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:24.290933563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:04:24.29641662Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBF7208BDD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:04:25.383727603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:04:25.389973862Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFBFB3350366, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:04:26.767269662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:04:26.770802228Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC00583BDBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:04:29.074832254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:29.870534678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:04:29.874123834Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC0BE7E857D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:04:29.874141995Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:29.874161027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:04:29.877033225Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC0BEAB5C4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:04:34.074944794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:34.284912174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:04:34.288413714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC1C59A6112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:04:34.288431109Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:34.28844906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:04:34.2913113Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC1C5C84F64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:04:35.861879306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:04:35.865261557Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC22397FDFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:04:35.931323813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:04:35.934654118Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC227BA9D9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:04:37.60450771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:04:37.607996933Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC28B780256, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:04:39.07473448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:40.170677548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:04:40.17399142Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC324696863, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:04:40.719743636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:04:40.723068912Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC34523C510, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:04:44.056175559Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:04:44.07478532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:04:44.074806614Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:04:44.074817729Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:04:44.07482588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:44.075137868Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-06-29T23:04:44.075148198Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:04:44.075167217Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.802µs success=true
level=info ts=2026-06-29T23:04:44.684294309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:04:44.687847196Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC43174CFE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:04:44.981614308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:04:44.985125407Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC4432D76D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:04:45.111821753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:04:45.115257977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC44AEFAEE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:04:47.994689516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:04:47.99821615Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC4F6C49FBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:04:48.330190311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:04:48.333707576Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC50AC3B6EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:04:49.074691253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:53.147691835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:04:53.151142542Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC629E94CBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:04:53.151159398Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:04:53.151177271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:04:53.154125676Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC62A17CC46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:04:53.810746121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:04:53.81416821Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC6516E6572, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:04:54.074902628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:04:54.993684695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:04:54.997842231Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC697F11412, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:04:58.892417862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:04:58.895849453Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC78052151A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:04:59.074760876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:02.372083539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:05:02.380274682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC85002777F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:05:02.404645792Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDAFC85176E723, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:05:04.007056141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:05:04.013102703Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC8B15571A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:05:04.075141965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:06.657481044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:05:06.660886656Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFC94F2752FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:05:09.07493235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:09.67547637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:05:09.678904019Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCA0309C9F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:05:11.456357102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:05:11.461829644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCA6D4F16D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:05:12.825091316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:05:12.831260372Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCABEEFD98A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:05:13.777986364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:05:13.781652103Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCAF79303D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:05:14.074944145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:05:14.07497755Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:05:14.074987705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:05:19.074707962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:05:24.074977852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:24.216241025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:05:24.220092859Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCD65C35195, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:05:25.436246289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:05:25.439661191Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCDAE754FA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:05:29.075109977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:33.988659409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:05:33.992004158Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFCFAC378957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:05:34.074962716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:36.16767986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:05:36.171124285Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD02E1AB14E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:05:36.544068093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:05:36.547321683Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD044869E8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:05:38.627149968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:05:38.630693357Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD0C0B3B51B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:05:39.075302909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:40.653440573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:05:40.65682419Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD13977980A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:05:43.762837586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:05:43.766101163Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD1F2CD8757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:05:44.055882294Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:05:44.074720964Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:05:44.07475235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:05:44.074743998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:44.151434553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:05:44.154782856Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD209F74FCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:05:44.838318682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:05:44.841641484Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD232E79220, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:05:47.582011608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:05:47.585443207Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD2D673C4CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:05:49.075345139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:51.973262241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:05:51.976782211Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD3DC302614, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:05:53.061287354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:05:53.069757734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD41D5638C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:05:54.074421274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:05:59.074878653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:05:59.259448996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:05:59.265101206Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD58E9BEE11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:05:59.267121299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:05:59.272170985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD58F091978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:05:59.272185744Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:05:59.272207573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:05:59.279436943Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD58F77F56B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:05:59.321499744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:05:59.326710204Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD592492A82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:06:00.901428422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:06:00.904828871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD5F05848A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:06:04.0749477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:06:09.074893936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:09.385269997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:06:09.391231018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD7EA2C7747, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:06:12.75204318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:06:12.760145809Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD8B2F8690D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:06:14.074665767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:06:14.074665663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:14.074698765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:06:16.986015273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:06:16.989270425Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFD9AF0E27BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:06:19.075166741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:21.411861615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:06:21.415246106Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDAB6DC692C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:06:22.69626672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:06:22.699824142Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDB036C5E70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:06:23.521133583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:06:23.5244935Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDB34951915, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:06:24.075270388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:25.715755477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:06:25.719096589Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDBB76463C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:06:29.074410151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:30.52408772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:06:30.52768381Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDCD5FF72D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:06:34.075021534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:35.042801253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:06:35.046038314Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDDE351ED69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:06:38.968674902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:06:38.972761402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDECD5DA212, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:06:39.074684393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:40.152584406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:06:40.158204489Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDF1406D6C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:06:41.26822897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:06:41.274159602Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDF568C77BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:06:43.898929175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:06:43.902393162Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFDFF332644E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:06:44.055655019Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:06:44.074809361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:06:44.074835962Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:06:44.074827425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:44.174454721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:06:44.177799749Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE0039D8C56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:06:45.554798745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:06:45.55823151Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE055E57E09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:06:47.777185548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:06:47.780575541Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE0DA5B62CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:06:48.90277474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:06:48.91134935Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE11DC30434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:06:48.911369801Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:06:48.911391325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:06:48.921742108Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE11E61D069, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:06:49.074763497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:06:51.020677183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:06:51.026558341Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE19BD4531D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:06:54.074476309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:06:59.07451318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:00.170092593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:07:00.173650568Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE3BD0A48BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:07:02.354440571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:07:02.363131517Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE43F8B7ACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:07:03.944018458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:07:03.950939109Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE49E2F0141, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:07:04.074797218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:07:09.074743259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:10.375832224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:07:10.379228056Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE61D574B71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:07:12.128744391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:07:12.132802065Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE685D6C4E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:07:14.074443735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:14.074451941Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:07:14.074481387Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:07:15.12058471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:07:15.123871412Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE73824B45F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:07:17.86953879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:07:17.872934178Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFE7DC000680, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:07:19.074934888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:07:24.074467473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:07:29.074546357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:29.229931553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:07:29.240315499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEA818D550D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:07:30.229891082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:07:30.237422684Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEABCFB121B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:07:30.441597957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:07:30.447135645Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEAC97A5FAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:07:30.778216615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:07:30.784323926Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEADD934C2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:07:32.564147575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:07:32.567507054Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEB47DDD975, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:07:33.40948513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:07:33.412805026Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEB7A4024F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:07:33.488164703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:07:33.491515627Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEB7EF08B58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:07:33.837278278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:07:33.840671038Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEB93C0A4FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:07:34.074417734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:35.367276544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:07:35.370591274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEBEEF21B54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:07:39.075015214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:40.194173296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:07:40.197682019Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFED0EA89C6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:07:40.506670711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:07:40.5102419Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFED2149FCF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:07:40.699835829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:07:40.703196213Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFED2CCB7ABE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:07:44.056557341Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:07:44.074915789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:07:44.074943969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:07:44.07495893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:07:49.075189635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:52.185263046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:07:52.189978824Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFEFD975ADFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:07:54.074351348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:07:55.372017792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:07:55.375674449Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF097566585, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:07:59.074739101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:08:04.074660274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:05.792590353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:08:05.797417251Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF3048698C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:08:06.395152317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:08:06.398656393Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF3285C0B98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:08:06.492931002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:08:06.496230183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF32E2D784B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:08:09.074394379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:13.117850446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:08:13.121160734Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF4B90EDAFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:08:14.074300844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:08:14.074333846Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:08:14.074334302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:14.676618696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:08:14.684857432Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF5163F73D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:08:14.825074897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:08:14.834914248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF51F32ECDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:08:15.587012679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:08:15.593804336Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF54C6C24FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:08:18.082092013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:08:18.085517856Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF5E0F2D042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:08:19.075426471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:21.680982021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:08:21.684384619Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF6B77495A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:08:23.545659093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:08:23.548996604Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF7269985C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:08:24.075337639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:24.519973491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:08:24.523310967Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF760AB89FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:08:29.074622451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:29.257519263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:08:29.260778783Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF87B0C5F84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:08:31.569808167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:08:31.573179573Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF904DF8740, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:08:32.992170302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:08:32.995424129Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF959A7BFF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:08:34.074982217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:34.992480672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:08:34.996105117Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFF9D0E481B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:08:38.368455074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:08:38.371982492Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFA9A1DB689, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:08:39.074426538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:44.055937538Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:08:44.074594219Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:08:44.07463743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:08:44.074679119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:46.106576674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:08:46.110316069Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFC675A2713, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:08:49.075331516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:51.228903524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:08:51.232405628Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFD98A6AEDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:08:52.963536514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:08:52.966846161Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFE000B1876, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:08:54.074839898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:08:56.296297674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:08:56.29992902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFEC6B22FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:08:56.663993129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:08:56.669444387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFEDCBB95B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:08:59.0748521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:00.336521787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:09:00.343936151Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDAFFFB7BDF2AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:09:04.074750693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:04.506267189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:09:04.509819186Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB000B00BF763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:09:04.789456402Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB000C0BB2B27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:09:05.949579211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.4 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:09:05.955703819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:09:05.957845849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0010659A2D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:09:05.961136377Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB001068EAFE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:09:09.075161793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:11.516444488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:09:11.519784492Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00251E01ACE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:09:13.515048044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:09:13.518373975Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB002C90134A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:09:14.074777535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:09:14.074818242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:09:14.074824101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:14.083399363Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-06-29T23:09:14.085256037Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009970922s
level=info ts=2026-06-29T23:09:14.377900072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:09:14.381262815Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB002FC6EBA3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:09:19.075170068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:21.869649619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:09:21.873010257Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB004BAF9FD7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:09:21.873031385Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:09:21.873061733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:09:21.875820499Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB004BB272ED6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:09:24.074592756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:24.623562533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:09:24.626885518Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0055F1FB7BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:09:28.038475753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:09:28.041852937Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0062AAADC24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:09:28.041870312Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:09:28.041889704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:09:28.04460964Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0062AD7B931, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:09:29.074901255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:30.921969697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:09:30.925271918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB006D6898869, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:09:33.40721777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:09:33.41056641Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0076AAAE308, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:09:34.07436256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:35.160646401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:09:35.163975516Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB007D32E7128, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:09:39.074572582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:39.836353766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:09:39.841790391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB008EA00635F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:09:41.024304673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:09:41.031933923Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00930F18209, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:09:43.568675768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:09:43.572158403Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB009C85A4D92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:09:43.736893501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:09:43.739999521Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB009D25B89BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:09:44.055578758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:09:44.074355235Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:09:44.074387443Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:09:44.074408076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:44.074430173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:09:44.074511229Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-06-29T23:09:44.074528863Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:09:44.074549366Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.067µs success=true
level=info ts=2026-06-29T23:09:44.809546978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:09:44.812996257Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00A124E1C1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:09:46.564942687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:09:46.568285964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00A7AEFE78D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:09:49.074647483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:49.350981083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:09:49.35919616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00B2148903E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:09:50.346598609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:09:50.352520065Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00B5C7ED346, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:09:52.200285197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:09:52.203870663Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00BCAD7B727, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:09:54.075066517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:09:59.074678376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:09:59.106842836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:09:59.110242084Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00D667EB426, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:10:01.772087873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:10:01.775680974Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00E055CEA34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:10:02.134646094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:10:02.138022759Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00E1AF6C525, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:10:04.074489882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:07.300702851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:10:07.304169987Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB00F4EE2AFA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:10:09.074992222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:13.513812289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:10:13.517165324Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB010C136DB4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:10:13.517181835Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:13.517202436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:10:13.519932799Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB010C16436DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:10:14.075035687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:10:14.07507452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:10:14.075104629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:15.157603697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:10:15.161100921Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01123336D84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:10:16.198538208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:10:16.201896318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB011613C33F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:10:16.814532043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:10:16.81782957Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01185F259F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:10:19.07473621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:10:24.07509201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:27.071303952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:10:27.075783076Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB013E95D5AC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:10:27.091068166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:10:27.096514575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB013EA9C2BC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:10:28.007964471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:10:28.015283217Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB014215EBDE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:10:29.074372716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:30.427115569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:10:30.430358718Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB014B152497E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:10:30.430376187Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:30.430394421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:10:30.433043617Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB014B17CFF68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:10:32.737588259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:10:32.740993616Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0153B0A7863, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:10:32.741011093Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:32.741031473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:10:32.74372693Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0153B372D29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:10:34.074396218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:34.966413515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:10:34.969857963Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB015BFE4AEE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:10:34.969876839Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:34.969896785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:10:34.972580333Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB015C0106671, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:10:35.270078502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:10:35.273325887Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB015D1FB2B04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T23:10:35.273342455Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:35.273361103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:10:35.276052808Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB015D2272C6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:10:36.543001247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:10:36.550970141Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0161E21EF9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:10:39.075036844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:43.49775199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:10:43.504992405Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB017BC9DBF0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:10:43.505016637Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:43.505039978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:10:43.508013723Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB017BCCF91B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:10:43.896258245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:10:43.899777525Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB017D4276392, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-29T23:10:43.899793993Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:43.899813984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:10:43.902760056Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB017D4559E85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:10:44.055982225Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:10:44.075261459Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:10:44.075291836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:10:44.075287304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:45.353673398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:10:45.357216681Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0182B06233D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:10:45.357236726Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:45.357259191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:10:45.360137322Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0182B35210E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:10:45.814257354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:10:45.817719484Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0184677B2AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:10:45.817738027Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:45.817757508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:10:45.820670265Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01846A67B91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:10:45.865041833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:10:45.868318441Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB018497DE8F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:10:47.051181399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:10:47.054615174Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0189032E95D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:10:47.054653545Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:47.054672301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:10:47.057477222Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0189060D557, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:10:48.092644521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:10:48.096143945Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB018CE459E29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:10:48.096167534Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:10:48.096195961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:10:48.099148357Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB018CE75EF03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:10:49.074523109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:49.401204042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:10:49.404676818Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0191C44E548, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:10:50.2085083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:10:50.211951197Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0194C62D79A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:10:54.076858065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:10:54.758128204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:10:54.761686055Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01A5B91B1F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:10:56.853260238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:10:56.85677337Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01AD874A292, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:10:58.398142428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:10:58.40152512Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01B3486E413, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:10:58.453661976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:10:58.456949136Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01B37D43B4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:10:59.07511294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:00.686504861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:11:00.690061395Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01BBCEDD0B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:11:02.49476247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:11:02.498419747Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01C28B7A9F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:11:03.393672192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:11:03.397122348Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01C5E484BF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:11:04.075006452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:05.105056133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:11:05.108706278Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01CC44BD887, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:11:07.069995271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:11:07.073994728Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01D3970EBFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:11:07.384655557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:11:07.388097493Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01D4C28BD4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:11:08.103995045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:11:08.10756199Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01D770B8FFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:11:08.881737364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:11:08.885208585Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01DA567D3D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:11:09.075317757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:10.637454374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:11:10.640802026Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01E0E0AF7C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:11:10.879673048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:11:10.883097315Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01E1C7C7F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:11:13.230538541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:11:13.234072049Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01EA89CFC2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:11:13.775678932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:11:13.779139351Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01EC91A9CF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:11:14.074784442Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:11:14.074816957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:11:14.074828983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:15.48608017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:11:15.489595173Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01F2F0D8E9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:11:16.808254505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:11:16.813892798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB01F7DFCB59E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:11:19.075028436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:21.887071312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:11:21.897943171Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB020AD0602D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:11:22.432648091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:11:22.438222551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB020CD3B7A8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:11:22.438243404Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:11:22.438262684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:11:22.440911741Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB020CD65F9DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:11:23.873333826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:11:23.876740332Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02122F7F1E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:11:24.074561173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:28.242741395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:11:28.246079875Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02227682802, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:11:29.074734986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:29.12490996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:11:29.128081976Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0225BFA03DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:11:30.659399014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:11:30.662758576Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB022B773459C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:11:31.527550652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="86 kB" avg_uncomp="86 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:11:31.531065323Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB022EB333534, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:11:32.813566209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:11:32.816753246Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02337D7307E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:11:33.333855375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:11:33.337012942Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02356D95446, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:11:34.075242628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:11:39.075120987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:42.820757259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:11:42.824139734Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0258C52D77E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:11:43.278587556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:11:43.28187955Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB025A79B67B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:11:44.056311431Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:11:44.074509516Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:11:44.074540954Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:11:44.074569529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:46.651104031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:11:46.654564861Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02670A1B393, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:11:48.958579639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:11:48.962435546Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB026FA302453, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:11:49.07458683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:50.956141936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:11:50.959482335Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0277139D056, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:11:50.959499324Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:11:50.959517571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:11:50.962248504Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02771679020, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:11:54.074399001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:55.647123466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:11:55.650399935Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02888D49F58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:11:56.813379684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:11:56.816724389Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB028CE581084, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:11:59.075145972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:11:59.635137293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:11:59.638481859Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02976892233, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:12:01.471061337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:12:01.474327297Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB029E3F692FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:12:02.759807133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:12:02.763246009Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02A30C9DA24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:12:03.268672123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:12:03.272009241Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02A4F1BA4AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:12:04.074982761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:04.159318589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:12:04.162826887Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02A84352376, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:12:05.910058437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:12:05.913347624Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02AEC8CC854, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:12:06.681574133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:12:06.690753428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02B1AE1F2E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:12:08.493885153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:12:08.500993303Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02B86C70625, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:12:09.074332884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:10.232929015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:12:10.238448087Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02BEE5731B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:12:10.238472678Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:12:10.238496043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:12:10.245907379Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02BEECA6981, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:12:11.148429391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:12:11.154289953Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02C24EF5794, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:12:13.57607864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:12:13.579501562Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02CB57C8F7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:12:14.074433154Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:12:14.074465893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:12:14.074464831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:15.565237642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:12:15.568755696Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02D2C0D5A4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:12:19.074455281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:21.579018683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:12:21.582256108Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02E927DB47C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:12:24.075358625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:27.227465157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:12:27.230817237Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB02FE32ACA01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:12:29.074544063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:29.510771031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:12:29.516785213Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0306B6C5CCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:12:30.115934183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:12:30.123398171Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0308F936C86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:12:33.239685872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:12:33.243200584Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0314988EEBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-29T23:12:34.074343317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:12:39.074893048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:39.354136785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:12:39.357757001Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB032B5FBD5A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:12:39.748832254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:12:39.752101578Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB032CD7F74A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:12:41.415833803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:12:41.419411732Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03330DEC2B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:12:44.055877425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:12:44.074295632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:12:44.074326641Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:12:44.074348853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:46.095704914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:12:46.099128816Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03447CEF1E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:12:47.41967845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:12:47.422952784Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03496B5F99D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:12:49.074677332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:12:49.987663632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:12:49.990995498Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0352FC741CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:12:50.737739316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:12:50.741057542Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0355C7CE369, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:12:52.359460914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:12:52.362814651Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB035BD26FBDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:12:53.684601911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:12:53.687871739Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0360C21E38F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:12:54.075143277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:12:59.074734214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:00.950602594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:13:00.953854272Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB037BD379C0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:13:01.25192775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:13:01.255153109Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB037CF2D5096, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:13:02.932024715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:13:02.935334493Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB038335399B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:13:04.074418488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:04.654239071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:13:04.663035989Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0389A4DA6D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:13:04.663056889Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:13:04.663080994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:13:04.674108034Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0389AF6E6BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:13:07.137062926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:13:07.145092617Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0392E3DDCCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:13:09.075048524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:13.636043642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:13:13.642079902Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03AB17E6479, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:13:14.074729243Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:13:14.074763334Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:13:14.074780267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:15.071014557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:13:15.074287583Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03B06DB4E47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:13:15.501981689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:13:15.505306061Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03B208BFFBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:13:18.377733353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:13:18.387835921Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03BCC5BE432, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:13:19.074586483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:19.579552933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:13:19.585621657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03C13C23983, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:13:24.074424518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:24.745016265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:13:24.748493816Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03D477C8B14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:13:25.747920003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:13:25.751206288Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03D834164FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:13:27.715843138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:13:27.719186576Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03DF88F1ED2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-29T23:13:29.075161677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:33.10505779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:13:33.108592788Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03F39C9B4D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:13:34.074707544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:36.018295595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:13:36.021643164Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03FE76BF870, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:13:36.236223712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:13:36.23935751Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB03FF46747E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:13:38.100267348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:13:38.103721839Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0406384761F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:13:38.807675628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:13:38.811033076Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0408DAEAD6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:13:39.075274882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:13:39.825621828Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB040CA2A05AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:13:44.056421276Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:13:44.074374853Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:13:44.074418787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:13:44.074438901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:47.878865752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:13:47.882217234Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB042AA5E82F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:13:47.897247968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:13:47.900470746Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB042AB75945A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:13:48.335667866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:13:48.338843232Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB042C59670B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:13:49.074937232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:49.649814326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:13:49.653062758Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04313EB5433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:13:54.075101305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:13:56.410316157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:13:56.709905859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:13:56.718154341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB044B906B376, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:13:57.307959474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:13:57.3115472Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB044DC64B777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T23:13:58.101744104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0450B7DFFCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:13:59.074748473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:00.459908335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:14:00.467806227Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0459885C806, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:14:02.329665208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:14:02.336174664Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04607E34334, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:14:04.075097754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:14:09.07495976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:09.183238781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:14:09.186662365Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB047A0352399, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:14:11.796680195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:14:11.80003436Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0483BFA80C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:14:14.074717055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:14:14.074749782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:14:14.074749452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:14.079953848Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-06-29T23:14:14.081836249Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007138134s
level=info ts=2026-06-29T23:14:17.896820655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:14:17.900261166Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB049A793FD02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:14:18.62705694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:14:18.630354628Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB049D31895BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:14:19.075338867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:19.99109572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:14:19.994440244Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04A2466674E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:14:20.262030106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:14:20.265141271Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04A3489CAEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:14:22.052607994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:14:22.055972797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04A9F474BB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:14:22.066219376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:14:22.069292637Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04AA0137062, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:14:24.075085322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:28.565240733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:14:28.568514184Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04C23752CFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:14:29.074485026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:30.328141557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:14:30.331543012Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04C8C8A7137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:14:32.392134545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:14:32.395372701Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04D078F0C35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:14:34.074893538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:38.12849612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:14:38.131932331Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04E5D7A0C8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:14:38.217133738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:14:38.220284058Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04E62BFCC33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:14:39.075187315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:42.102097706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:14:42.105356742Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04F4A51686C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:14:44.046387031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:14:44.049656776Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB04FBE33A0FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:14:44.055705881Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:14:44.074163748Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:14:44.074314999Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-06-29T23:14:44.074322739Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:14:44.074330544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:14:44.07434363Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.169µs success=true
level=info ts=2026-06-29T23:14:44.074365976Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:14:44.074408102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:45.193462219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:14:45.197004799Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0500296DAF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:14:46.582748418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:14:46.586145644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05055646CA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:14:49.075406876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:50.584200628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:14:50.587726316Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05143E59723, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:14:51.67066767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:14:51.676111342Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05184C6DF59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:14:52.514435989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:14:52.521621948Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB051B72C58AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:14:54.07512031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:14:55.558378381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:14:55.566732981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0526CACA891, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:14:59.075222561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:01.761278712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:15:01.767138751Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB053DE40B95A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-06-29T23:15:04.075017643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:04.706828926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:15:04.710454741Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0548DAEF79B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:15:04.760799014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:15:04.76568474Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05490FA247F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:15:05.435658829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:15:05.441398965Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB054B94004CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:15:09.074807535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:11.169912177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:15:11.173266475Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0560EE61447, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:15:13.650612904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:15:13.654032943Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB056A2C2EA87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:15:14.075116651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:15:14.075149259Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:15:14.075213392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:14.158613178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:15:14.161916968Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB056C10921AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:15:15.047994354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:15:15.05131881Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB056F60B9A79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:15:18.423259652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:15:18.426654907Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB057BF3B9EFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:15:19.074651838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:23.04549638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:15:23.050999507Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB058D2DDE8AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:15:24.039669149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:15:24.045298058Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0590E22A5CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:15:24.074506827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:27.861577433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:15:27.864976969Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB059F1CD463C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:15:29.075310842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:29.61113458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:15:29.614691033Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05A5A16DC52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:15:30.382664741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:15:30.385853843Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05A88102E6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:15:33.857668478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:15:33.861005856Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05B57311E99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:15:34.074429139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:34.724691643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:15:34.72786827Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05B8ADD567E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:15:36.318366827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:15:36.326093448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05BEA1FD7D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:15:39.074615276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:39.317076162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:15:39.324081313Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05C9CD232B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:15:42.032293518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:15:42.037821393Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05D3E91EA41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:15:44.055885083Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:15:44.074807201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:15:44.074839966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:15:44.074839993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:45.460442584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:15:45.466552128Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05E0AEF3F47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:15:49.075022284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:52.125121417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:15:52.128484432Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05F98054361, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:15:53.155670831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:15:53.160539995Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB05FD588D74C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:15:54.074669224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:15:54.442145396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:15:54.448013433Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB060224544FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:15:55.189087844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:15:55.192516822Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0604EA6D362, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:15:56.745870074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:15:56.749048005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB060AB6D283A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-06-29T23:15:58.462734214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:15:58.466166016Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06111C6652A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:15:58.466186125Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:15:58.466207646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:15:58.468916004Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06111F30AE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:15:59.074621411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:02.836492739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:16:02.839873059Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0621677DE9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:16:02.839888643Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:16:02.839910125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:16:02.842739302Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06216A4E05A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:16:03.406354749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:16:03.409717361Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB062386E6BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:16:04.074613259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:05.747861486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:16:05.751212626Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB062C3FECFD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:16:06.450980106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:16:06.454319564Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB062EDE7C604, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:16:09.074719387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:09.093827498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:16:09.097277636Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0638B6FE36A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:16:12.811197581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:16:12.81466426Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0646902C9E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:16:12.902972032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:16:12.906258528Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0646E7960AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:16:14.075148747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:16:14.075185328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:16:14.075236672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:17.832561224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:16:17.836044776Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB065944F67F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:16:18.50903862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:16:18.512681051Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB065BC9EC8C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:16:19.075429559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:16:24.074519005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:24.376918363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:16:24.380344781Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0671A60A5CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:16:26.494020591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:16:26.502066579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06798D889A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:16:29.075338528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:29.156649048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:16:29.162872923Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB068377010F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:16:29.936403502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:16:29.93980289Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06865BF694B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:16:32.416035609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:16:32.42205099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB068F9B43CB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:16:32.889524758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:16:32.89686366Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0691600EA56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:16:34.075311438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:16:39.074737981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:41.732472643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:16:41.735749213Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06B24D76C13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:16:42.048550493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:16:42.052022187Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06B37B15066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:16:44.056394494Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:16:44.074663717Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:16:44.07467708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:44.074701032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:16:44.275432096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:16:44.280462209Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06BBC856C09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:16:46.01839769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:16:46.025033942Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06C2480601F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:16:46.763498615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:16:46.766866148Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06C50B89B5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:16:46.908258106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:16:46.911664503Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06C595A3F49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:16:46.911682996Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:16:46.911704054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:16:46.914467609Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06C5987DD44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:16:49.074944741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:49.461085528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:16:49.46433678Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06CF180A858, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:16:49.464362074Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:16:49.464386709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:16:49.467211871Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06CF1AEAC2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:16:50.772918638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:16:50.776232763Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06D3FB30148, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:16:51.046787109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-06-29T23:16:51.050220105Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06D50074DDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-06-29T23:16:51.050242262Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:16:51.050264423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:16:51.053261875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06D5037F538, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-06-29T23:16:54.074588156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:58.960693085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:16:58.964254401Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06F27BCFD20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:16:58.964280889Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:16:58.964308908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:16:58.967118814Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06F27EB6080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:16:59.075154489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:16:59.886910525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:16:59.893903039Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06F5EF042A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:17:00.018000614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:17:00.021313854Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB06F66BED9DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:17:02.85555996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:17:02.858918456Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0700FE2232C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:17:04.07438172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:08.4630254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:17:08.466415931Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0715E1D5AA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:17:08.594655834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:17:08.597951036Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07165F4980B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T23:17:08.597967442Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:08.696290877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:17:08.699592821Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0716C040652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:17:08.898272676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:17:08.901910338Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB071780ED8BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:17:09.075160656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:09.239473268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:17:09.243036618Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0718C664BC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:17:12.341579656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:17:12.344982265Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB072454AFB4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:17:12.345001937Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:12.3450219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:17:12.347997385Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB072457CACD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:17:14.074528888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:17:14.074525884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:14.074559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:17:15.413269441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:17:15.41669688Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB072FC617538, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:17:16.184912623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:17:16.190436185Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0732A80DFBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:17:16.443027652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:17:16.448749066Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07339E542E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:17:18.320556215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:17:18.328493181Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB073A9F09442, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:17:19.074414691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:17:24.074817501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:24.704746611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:17:24.710864737Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB075265CBDFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:17:24.722895706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:17:24.725950915Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0752743A105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:17:24.853385182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:17:24.856746165Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0752F0D600E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:17:26.037270546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:17:26.04079411Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07575A0DFB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:17:27.222046709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:17:27.225591545Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB075BC3EE22D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:17:28.905818431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:17:28.907016214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:17:28.90923368Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0762099D136, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:17:28.909253647Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:28.909273878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:17:28.910735652Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07620B277E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:17:28.910748431Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=error ts=2026-06-29T23:17:28.912056118Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07620C91008, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:17:29.074862561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:29.210202227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:17:29.213462569Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07632BD3983, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:17:30.28194533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:17:30.285289707Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB076729F3BEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:17:30.285307428Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:30.285327285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:17:30.287916208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07672CB0495, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:17:31.17940739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:17:31.182767915Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB076A81E6708, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:17:31.182787807Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:31.182807394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:17:31.185507214Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB076A84A7FE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:17:34.075068639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:34.486356924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:17:34.489603629Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0776D38480B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:17:36.055295586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:17:36.058411566Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB077CABACE3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:17:37.27406656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:17:37.279023339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB078137A9FE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:17:39.07468755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:40.061037283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:17:40.064347024Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB078B97F12F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:17:40.65146827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:17:40.654935364Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB078DCB323F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:17:41.499375458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:17:41.502768727Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0790F3C1D91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:17:41.54579363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:17:41.548895392Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07911FC926B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:17:44.05617798Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:17:44.074744338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:17:44.074772989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:17:44.07479182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:45.702216549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:17:45.705863596Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07A09C276B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:17:45.705881142Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:45.705901183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:17:45.708647267Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07A09EF7EF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:17:46.976120335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:17:46.979737054Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07A55AE2264, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:17:47.504415351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:17:47.508031182Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07A752B38FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:17:47.508049253Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:17:47.508068586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:17:47.510859671Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07A755990C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:17:49.07487266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:17:54.075180021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:17:59.054606233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:17:59.059253982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07D25AEB720, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:17:59.074339586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:00.380005728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:18:00.383456047Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07D749D414B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:18:00.383473649Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:18:00.38349159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:18:00.38635156Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07D74CAD2E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:18:01.44061367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:18:01.444117983Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07DB3D4E411, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:18:02.999675833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:18:03.003000494Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07E10C04CEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:18:03.232679391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:18:03.236075213Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07E1EA3DDAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:18:03.293385397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:18:03.296797197Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07E2241E917, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:18:03.43918008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:18:03.445539221Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07E2B200DBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:18:04.074347936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:07.087670718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:18:07.095454351Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F04ADA054, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:18:07.634871321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:18:07.638109924Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F2506368A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:18:08.964163322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:18:08.967515506Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F74430954, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-29T23:18:08.96753415Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:18:08.967555248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:18:08.970413279Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F7471CDD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:18:09.075160563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:09.491991127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:18:09.495303017Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F93B8112D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:18:09.495318784Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:18:09.495336986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="18 kB" avg_uncomp="18 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:18:09.498182017Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB07F93E6C1CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:18:11.356736063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:18:11.365177872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB080032B1395, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:18:12.700650793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:18:12.706837371Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0805323FB61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:18:13.222691822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:18:13.226042538Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08072172DAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-29T23:18:14.074731411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:14.074763909Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:18:14.074781775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-06-29T23:18:14.378171436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB080B6C4DCA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:18:14.704286509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:18:14.707728485Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB080CA672437, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:18:15.337883673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:18:15.341127548Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB080F028B699, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:18:15.724007824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:18:15.72743654Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB081072F0ACC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:18:18.280965219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:18:18.284405702Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0819F97C2C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:18:19.075214459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:18:24.074981697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:25.976774907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:18:25.980219282Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0836A4C04A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:18:26.303159961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:18:26.306448753Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0837DBDBE57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:18:26.62020751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:18:26.623555289Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08390A467B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:18:26.624531595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:18:26.627486763Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08390E20BA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:18:28.466475183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:18:28.469935882Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB083FEB18311, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:18:28.566139849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:18:28.571234923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08404BBD3CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:18:29.075141346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:29.484042961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:18:29.490842085Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0843B8C6DCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:18:34.074957436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:18:39.075347139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:44.055622917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:18:44.074906875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:18:44.074939037Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:18:44.074957622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:48.183478263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:18:48.187059711Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08895ECBBC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:18:49.075030078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:49.973675243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:18:49.977214867Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08900A08B31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:18:51.728477762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:18:51.734119069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0896959E5BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:18:51.734137414Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:18:51.734159493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:18:51.742335888Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08969D72F11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:18:52.61368577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:18:52.619764594Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0899E22D6DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:18:53.482319442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:18:53.485725752Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB089D1BF5D28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:18:54.074537922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:56.548139118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:18:56.551406478Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08A887C2770, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:18:56.953672494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:18:56.956988766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08AA0A74702, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:18:58.823397925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:18:58.82658941Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08B1018C59D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:18:59.075256877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:18:59.314050219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:18:59.317265686Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08B2D582E5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:18:59.319307868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:18:59.32205562Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08B2DA2024F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:19:01.051669541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:19:01.055079074Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08B94EBE75A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:19:03.162678211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:19:03.168838928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08C12E83202, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:19:04.075275143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:04.583538405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:19:04.58970318Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08C6799BB16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:19:05.03834569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:19:05.047222284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08C82E0E0DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:19:09.075141479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:13.251516154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:19:13.257610555Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08E6C403131, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:19:13.854217518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:19:13.859064155Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08E90196596, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:19:14.074834325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:19:14.07483704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:14.074870155Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:19:14.080954305Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-06-29T23:19:14.082887881Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008405958s
level=info ts=2026-06-29T23:19:14.351885771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:19:14.357643949Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08EADD07157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:19:17.119077124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:19:17.122732322Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08F529F96F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:19:18.021516584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:19:18.024933819Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08F8866CFE7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:19:19.075441685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:19.770454337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:19:19.773756368Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB08FF0A4CEA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:19:21.709364483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:19:21.71275402Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0906435FB67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:19:23.187274194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:19:23.193254434Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB090BC76B1B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:19:24.075177808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:19:29.075257586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:30.010433868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:19:30.019346746Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0925350BD18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:19:34.029189811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:19:34.032688232Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB093428A14BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:19:34.074467287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:19:39.074736572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:39.190066215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:19:39.193433006Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09476263E4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:19:40.484678817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:19:40.488103028Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB094C350AAA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:19:44.055887395Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:19:44.07473205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:19:44.074745421Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:19:44.074760749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:19:44.074788569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:44.074888512Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-06-29T23:19:44.074897501Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:19:44.074915241Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.691µs success=true
level=info ts=2026-06-29T23:19:48.723104994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:19:48.726387966Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB096AE5BF062, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:19:49.074013645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:19:49.075096385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:19:49.077260433Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB096C34591AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:19:50.10068677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:19:50.104168002Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09700794D8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:19:51.299992052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:19:51.30362276Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09747F70035, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:19:54.075043833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:19:54.84219043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:19:54.845789009Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0981B18DC03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:19:55.061673619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:19:55.06499675Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09828291C71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:19:57.811205252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="256 kB" avg_comp="256 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" max_age=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:19:57.819176704Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB098CC542A02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:19:58.112092572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="582 kB" avg_comp="582 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" max_age=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:19:58.126064697Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB098DE9BE6FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:19:58.41176625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="129 kB" avg_comp="129 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" max_age=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:19:58.41886238Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB098F00096A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:19:59.075037129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:00.934205449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:20:00.937793764Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09986356941, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:20:01.195092969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:20:01.20422166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0999617760C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:20:01.204245703Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:20:01.204272874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:20:01.215129073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09996BD776F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:20:01.510613242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:20:01.516936668Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB099A8B98323, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:20:04.074978751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:04.891337148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:20:04.896270874Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09A722710B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:20:05.941751218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:20:05.947848645Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09AB0D45F40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:20:06.347889893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:20:06.351516498Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09AC8E4A3C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:20:06.459746633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:20:06.463129332Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09ACF8BBB52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:20:06.735842395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:20:06.739192697Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09AE000F4F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:20:08.795361593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:20:08.800257167Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09B5AC73FBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:20:09.075167453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:13.056045188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:20:13.064357437Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09C5901B012, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:20:13.979963622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:20:13.985948731Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09C8FEFD820, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:20:14.07488008Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:20:14.074911677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:20:14.074927185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:14.134291774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:20:14.138387859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09C99076203, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:20:16.474849592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:20:16.481272278Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09D24ABF83D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:20:17.238099906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:20:17.245576179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09D523B5E8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:20:19.074977636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:21.248175105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:20:21.253894308Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09E41254B50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:20:23.465841338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:20:23.469157805Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB09EC52FDFCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:20:24.074524621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:20:29.074507055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:29.840608205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:20:29.844109134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="146 kB" avg_comp="146 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" max_age=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:20:29.845187538Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0412973A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:20:29.847700558Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A04159EC6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:20:30.144987189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="171 B" avg_comp="171 B" total_uncomp="24 kB" avg_uncomp="24 kB" max_age=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:20:30.148284552Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0534AE0C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:20:30.444742437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="72 kB" avg_uncomp="72 kB" max_age=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:20:30.4480963Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0652AB4F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:20:30.744880074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="102 B" avg_comp="102 B" total_uncomp="15 kB" avg_uncomp="15 kB" max_age=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:20:30.748279737Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0770D3F67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:20:31.851297436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:20:31.854600708Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0B8FEC18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:20:31.919814803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:20:31.92307634Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0BD14E250, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:20:32.654040607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:20:32.657397113Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A0E8D88369, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:20:34.075192035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:37.296466015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:20:37.299905288Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A1FD8F0171, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:20:38.909946077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:20:38.913286266Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A25DBA2B57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-06-29T23:20:39.07434811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:39.081467059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:20:39.084604228Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A267F0B2AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:20:39.694905893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:20:39.698519488Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A28C88775C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:20:39.853969453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:20:39.865498638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A2967B9424, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:20:39.941581232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:20:39.947462471Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A29B5E6BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:20:40.569838331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:20:40.574010292Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A2C0B62CC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:20:44.055575762Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:20:44.074856193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:20:44.074885335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:20:44.074893997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:46.481755666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:20:46.488009562Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A42135A95C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:20:47.759015366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:20:47.762468391Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A46D2CFFD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:20:48.783599586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:20:48.786913009Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A4AA3E050A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:20:48.952463115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:20:48.95569929Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A4B44D2C1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:20:49.075247265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:49.258856588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:20:49.262029236Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A4C68F8CFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:20:49.296078034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:20:49.299866042Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A4C8CC4156, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:20:50.106880257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:20:50.110197876Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A4F91CD14D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:20:50.75859285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="922 B" avg_comp="922 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:20:50.761858045Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A51FF6054C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-06-29T23:20:53.550486377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:20:53.554175839Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A5C66529D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:20:54.075005325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:54.978670476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:20:54.982028992Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A61B7F80F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:20:58.143745131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:20:58.148765108Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A6D83F2CDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:20:59.075049008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:20:59.088193181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:20:59.094010922Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A710982690, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:21:01.197332223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:21:01.200662018Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A78E281B8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-06-29T23:21:04.074753313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:06.737565803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:21:06.741041511Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A8D862091B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-06-29T23:21:09.07523563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:11.303895751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:21:11.310061291Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0A9E8BA06D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:21:12.728806807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:21:12.735391362Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AA3DAE6EAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:21:14.074594812Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:21:14.07464027Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:21:14.074694166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:15.087204815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:21:15.090409244Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AACA0D3CF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:21:16.295027511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:21:16.298325979Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AB120BFD42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:21:19.074923115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:19.218287643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:21:19.223010777Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0ABC04D626C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:21:22.428669134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:21:22.432603665Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AC7FA96DC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:21:23.62023508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:21:23.628134677Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0ACC6F0212D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:21:24.074808273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:25.1176338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:21:25.12587838Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AD20352DE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:21:25.624309176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:21:25.630242999Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AD3E45AC4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:21:26.690352974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:21:26.694824328Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AD7DA981A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:21:26.764996237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:21:26.768219868Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AD821B3622, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:21:27.058293115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:21:27.06181481Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AD9399133C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:21:28.21176718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:21:28.215746904Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0ADD860EB59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:21:28.250814144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:21:28.256813519Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0ADDAD2DE14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:21:29.074883094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:30.564148786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:21:30.56782427Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AE6490F87D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:21:30.601867747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:21:30.605152026Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AE66CCD363, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:21:30.977239826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:21:30.980659083Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AE7D2D7175, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-06-29T23:21:34.074895723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:34.579464169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="320 B" avg_comp="320 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:21:34.583890411Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AF53E56E9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:21:36.552200968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:21:36.555453284Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0AFC9780026, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:21:38.056469372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:21:38.059897877Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B02322D96B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-06-29T23:21:39.075070493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:40.652188651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:21:40.655715247Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B0BDDAB884, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:21:41.96122604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:21:41.96472961Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B10BE15FC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:21:43.200280011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:21:43.203754615Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B155BAB465, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:21:44.055598967Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:21:44.074655665Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:21:44.07468685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:21:44.074690112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:46.847520239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:21:46.850951402Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B22F1F557F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:21:49.074584858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:52.258839995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:21:52.263883282Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B371C128C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:21:52.263904151Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:21:52.263927747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:21:52.271846304Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B3723D782E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:21:52.957940483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:21:52.963545715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B39B783FEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-06-29T23:21:54.074968411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:21:57.365676064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:21:57.369421488Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B4A2111657, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=warn ts=2026-06-29T23:21:59.074657194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:01.751022652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:22:01.757171692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B5A79BAC59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:22:03.061673007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:22:03.067937665Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B5F5BC666D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-06-29T23:22:04.075004738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:04.441462248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:22:04.446434063Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B647E7AD49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:22:08.811545535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:22:08.817653119Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B74C7131BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:22:09.075047676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:10.2859564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:22:10.289158011Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B7A4275D67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:22:11.135643503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:22:11.139807699Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B7D6D961FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:22:14.050672172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:22:14.05620788Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B884AEF61E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:22:14.074787688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:22:14.074818357Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:22:14.074833704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:15.762205725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:22:15.765543158Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B8EA92257D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:22:16.019092718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:22:16.028691918Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B8F9E028E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:22:16.075234239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:22:16.084425924Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B8FD92728E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:22:16.302944701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:22:16.30899628Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B90AF50666, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:22:16.623232751Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB0B91DB2E72D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:22:16.81467316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:22:16.81866334Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B9294FEEA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:22:17.023214166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:22:17.026504624Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B935BA9420, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:22:17.277956809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:22:17.283231415Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B944EB71B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:22:19.075055106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:19.218404244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:22:19.221864003Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B9B89368B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:22:20.341085193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:22:20.344438981Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0B9FB7DC042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:22:21.053353243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:22:21.057281529Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BA25F98EDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:22:23.715676484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:22:23.722051928Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BAC4D2198C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:22:24.074998721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:24.27898711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:22:24.28221703Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BAE6359441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:22:26.877148847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:22:26.880516291Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BB8113478F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:22:29.075335938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:30.879952743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:22:30.885619008Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BC6FCB7870, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:22:33.895431261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:22:33.901507327Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BD238F6554, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:22:34.075072764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:34.194420585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="940 B" avg_comp="940 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:22:34.197952279Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BD353A0672, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-06-29T23:22:34.197970249Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:22:34.197991073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="80 kB" avg_comp="80 kB" total_uncomp="689 kB" avg_uncomp="689 kB" max_age=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:22:34.201328432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BD356A5459, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:22:37.688990924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:22:37.692483961Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BE05848169, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:22:39.075176971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:39.723925294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:22:39.727467138Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BE7ED0CA87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:22:39.727484583Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:22:39.727503861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:22:39.730359983Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BE7EFE8F3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:22:42.153036459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:22:42.156403594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BF0F96744F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:22:43.119912429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:22:43.12325598Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BF49381304, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:22:43.912780487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:22:43.91628865Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0BF787C1428, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-06-29T23:22:44.056176451Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:22:44.07509573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:22:44.075127019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:22:44.075144222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:48.699826584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:22:48.703432618Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C095CF4016, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:22:49.075277337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:49.81984631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:22:49.825783306Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C0D8B5E4A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:22:54.074915441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:22:54.97001248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:22:54.975787732Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C20BADE234, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:22:56.410033551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:22:56.413361038Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C2615E77AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:22:56.940980559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:22:56.94428225Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C28104195C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-06-29T23:22:57.171107809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:22:57.174392347Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C28EBA955B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:22:58.564741221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:22:58.567968186Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C2E1CB5B33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-06-29T23:22:59.074666587Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:00.277311657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:23:00.281483923Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C347EDA9F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:23:00.378678783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:23:00.384660051Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C34E1456F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-06-29T23:23:03.445467819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:23:03.453498328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C404FFD83D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:23:04.074562074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:05.231139486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:23:05.236959476Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C46F4BBE5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:23:05.236981366Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:05.237002303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:23:05.239740359Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C46F78D652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:23:05.7779323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:23:05.781204152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C48FBD9EFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-06-29T23:23:06.790167423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:23:06.793487543Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C4CC13D32A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:23:06.793507729Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:06.793527738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:23:06.796378517Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C4CC41D9A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:23:07.639788016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:23:07.644382822Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C4FEBA22FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:23:09.075148946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:10.832539669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:23:10.835895469Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C5BD049785, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:23:12.592749591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:23:12.596087007Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C625F001CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:23:14.07455738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:23:14.07458772Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:23:14.074603454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:14.461487764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:23:14.464678422Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C6954FD4C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:23:15.089865763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:23:15.09315047Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C6BAC640C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:23:15.093166173Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:15.093183323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:23:15.095877791Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C6BAF1F400, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:23:15.783955627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:23:15.787210661Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C6E4255CF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:23:15.870579121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:23:15.874348393Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C6E951247C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:23:17.911192747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:23:17.914882705Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C762F5FB1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:23:19.074997336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:19.37346185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:23:19.382827575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C7BA718AAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:23:21.367581421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:23:21.373826251Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C8311C460E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:23:22.153983938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:23:22.157407794Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C85FD4AAA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:23:22.789675295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:23:22.793092039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C885B8CAFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:23:22.879446987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:23:22.882899823Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C88B1323D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:23:23.262017865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:23:23.265586978Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C8A1E1D185, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:23:24.074554295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:25.338726872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:23:25.342023843Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C91DA6460E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:23:25.587998964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:23:25.591102148Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C92C80CAAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:23:27.779400355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="385 kB" avg_uncomp="385 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:23:27.783000933Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C9AF24611D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-06-29T23:23:27.783021845Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:27.78304204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="158 B" avg_comp="158 B" total_uncomp="223 kB" avg_uncomp="223 kB" max_age=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:23:27.786059019Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0C9AF55A69D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:23:29.074776903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:33.191167519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:23:33.194649582Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CAF1B39961, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:23:34.074592765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:23:39.07502014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:40.117959441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:23:40.121376464Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CC8E9204EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-06-29T23:23:40.985523916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:23:40.988826878Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CCC24627D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-06-29T23:23:41.321824194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="338 B" avg_comp="338 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:23:41.325203162Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CCD652FCA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-06-29T23:23:41.325221629Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:41.32524163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="70 kB" avg_comp="70 kB" total_uncomp="655 kB" avg_uncomp="655 kB" max_age=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:23:41.328752533Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CCD684C865, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:23:44.055885748Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:23:44.074482937Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:23:44.074523726Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:23:44.074542538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:44.673831253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:23:44.681931585Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CD9E666AE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:23:45.733670297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:23:45.740189646Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CDDD79FE67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:23:45.740210051Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:45.740232633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:23:45.742931016Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CDDDA60BE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:23:46.92374116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:23:46.929279073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CE24598648, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:23:47.705754249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:23:47.712581117Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CE530A8B2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:23:47.754760325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:23:47.76041374Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CE55E4D4A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:23:49.074644079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:51.789880406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:23:51.793262371Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CF46448D87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:23:51.890577227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:23:51.893861298Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CF4C443871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:23:51.994059122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:23:51.997328031Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CF526F488C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:23:51.997345442Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:51.997363656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:23:52.000066561Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CF529C05EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:23:53.95967522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:23:53.963388602Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0CFC79B02B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:23:54.074496366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:23:55.044882994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:23:55.048088719Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D008455E3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:23:56.439306804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:23:56.442680124Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D05B644DF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:23:56.632192823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.5 kB" avg_comp="3.5 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-06-29T23:23:56.635455107Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D066E3C5E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-06-29T23:23:56.635472927Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:23:56.635496017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="120 kB" avg_comp="120 kB" total_uncomp="774 kB" avg_uncomp="774 kB" max_age=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:23:56.640866938Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D06730F94E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:23:57.823509817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:23:57.826936078Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D0ADE76A08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:23:58.413738317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:23:58.417072873Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D0D114727D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:23:58.570677202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:23:58.573844021Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D0DA6DA8CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:23:59.075218306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:01.999733291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:24:02.004312934Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D1A6D579D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:24:02.913989253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-06-29T23:24:02.917355629Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D1DD5204E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-06-29T23:24:02.917372402Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:02.917394026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="351 kB" avg_comp="351 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" max_age=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:24:02.925191282Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D1DDCB02BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:24:04.075063788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:04.74657794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:24:04.75473924Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D24AD4594E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:24:05.043668428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:24:05.049309506Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D25C64DF9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:24:06.188811264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:24:06.192427548Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D2A0876AD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:24:06.618464185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:24:06.621912876Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D2BA206D2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:24:09.074509847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:09.648673054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:24:09.652095987Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D36EBE0247, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:24:09.652113658Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:09.652132894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:24:09.655071312Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D36EEE0240, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:24:12.206289002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:24:12.210085552Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D4073029FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:24:14.074880651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:24:14.074900736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:14.074914054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:24:14.081138553Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-06-29T23:24:14.083219797Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008187471s
level=info ts=2026-06-29T23:24:14.817040902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:24:14.820388985Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D4A2CACE99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:24:14.961898105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:24:14.965773015Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D4AB6FAADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:24:15.424234329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:24:15.428058074Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D4C6FE925B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:24:16.173215106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:24:16.178135413Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D4F3B94CD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:24:18.999846311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:24:19.00574887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D59C42827A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=warn ts=2026-06-29T23:24:19.075296029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:19.426195721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:24:19.42937102Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D5B58392FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:24:19.42938553Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:19.429402774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:24:19.432057698Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D5B5AF2375, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:24:22.205932594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:24:22.211010686Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D65B4E3718, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:24:24.075335723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:24.091504502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:24:24.097480858Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D6CBC07F7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:24:27.439888916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:24:27.443706195Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D793306438, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:24:28.720444109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:24:28.723885737Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D7DF80A7AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-06-29T23:24:28.723907538Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:28.723929877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="465 B" avg_comp="465 B" total_uncomp="262 kB" avg_uncomp="262 kB" max_age=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:24:28.727078727Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D7DFB3E231, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:24:29.074890007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:30.887043153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:24:30.890264292Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D860A280A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:24:31.369553981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:24:31.373667282Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D87D714E2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:24:31.381665062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:24:31.391789937Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D87E88A4CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:24:31.786967846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:24:31.793517654Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0D89675EAFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:24:34.07451992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:38.795441882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:24:38.799226509Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DA3808A318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:24:39.075278563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:39.248675005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:24:39.251941505Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DA53068517, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-06-29T23:24:39.251961865Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:39.251981521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="118 B" avg_comp="118 B" total_uncomp="208 kB" avg_uncomp="208 kB" max_age=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:24:39.254962567Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DA5337892D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:24:44.055871982Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:24:44.074258738Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:24:44.074387563Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-06-29T23:24:44.074399903Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:24:44.074407828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:24:44.074424202Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=18.822µs success=true
level=info ts=2026-06-29T23:24:44.074432718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:24:44.074518815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:44.18294787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:24:44.186370911Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DB79254E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:24:44.431325458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:24:44.435307725Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DB87F3CABF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:24:44.947021893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:24:44.950375534Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DBA6AE2274, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:24:45.109303621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:24:45.113266306Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DBB061F050, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:24:45.140067813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:24:45.149071567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DBB2859215, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:24:46.370494266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:24:46.376592326Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DBFBAF6994, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:24:46.977237989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:24:46.981274343Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DC1FB9CC24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:24:47.897062418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:24:47.904762612Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DC56B4347F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:24:48.741670113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:24:48.745034069Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DC88DCA06D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:24:49.075122527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:50.243377608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:24:50.246751162Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DCE25DEFBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:24:52.128487772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:24:52.131901687Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DD52BB7C7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:24:52.928751545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:24:52.932123543Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DD826E060A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:24:54.074882944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:24:54.170316348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:24:54.173801694Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DDCC700172, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:24:54.173818158Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:24:54.173835778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:24:54.176614753Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DDCC9DFA27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:24:54.384605398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:24:54.387886499Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DDD9337B2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:24:55.479820424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:24:55.483744435Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DE1A8090AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:24:56.492907783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:24:56.496298052Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DE56DFBDC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:24:59.075396167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:00.66587428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:25:00.669685177Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DF4F9C9CE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:25:01.222076438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:25:01.225576777Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DF70C0F3C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-29T23:25:01.225594545Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:25:01.225616124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:25:01.228693794Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0DF70F248A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:25:04.07502727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:05.228675718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:25:05.232756069Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E05F942506, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:25:06.28734071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:25:06.290869974Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E09EACFE49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:25:09.075126202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:10.806414601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:25:10.809858473Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E1AC06CCCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:25:11.869134033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:25:11.87280904Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E1EB611A13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:25:14.008252708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:25:14.01254973Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E26AEBE0D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:25:14.035567796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:25:14.042603042Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E26CB762D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:25:14.07451858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:25:14.07452174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:14.074542353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:25:14.471290141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:25:14.476141901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E2868EE1E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:25:16.326993652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:25:16.33270124Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E2F535CE86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=info ts=2026-06-29T23:25:17.844290737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:25:17.847756134Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E34F835AB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:25:19.075010404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:22.963230956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:25:22.966999745Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E480A1998C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:25:23.964757012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:25:23.968233236Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E4BC527A9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:25:24.075325208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:25.233147823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:25:25.236541761Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E507EBB2C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:25:25.705531187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:25:25.714440384Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E52468A138, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:25:26.164800664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:25:26.171225273Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E53FA2C927, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:25:28.577738905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:25:28.583460197Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E5CF6A677A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:25:29.074505256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:31.740912391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:25:31.745967837Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E68BD53C42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:25:32.666680165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:25:32.670127656Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E6C2FF0007, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:25:34.075217125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:34.34823721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:25:34.351670665Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E727390C0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:25:35.940395435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:25:35.944327492Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E78627867A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:25:37.382861928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:25:37.389385198Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E7DC471DFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:25:39.021983641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:25:39.025918471Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E83DCEBA00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:25:39.07445685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:42.120303749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:25:42.123698008Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E8F6796740, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:25:42.274203047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:25:42.279767504Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E8FFC66DA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:25:42.279789676Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:25:42.279815563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:25:42.287515669Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E9003E1437, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:25:42.598316082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:25:42.604355529Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E9131F2BA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:25:43.894534279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:25:43.89803044Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E9603AE3AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:25:43.935897567Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB0E9627FE2B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:25:44.056311544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:25:44.07510493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:25:44.0751397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:25:44.075134445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:44.565009218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:25:44.568286078Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0E988306668, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:25:49.074616601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:49.298998553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:25:49.302453568Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EAA25DD646, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:25:51.400876576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:25:51.404787178Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EB1FA7F9E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:25:51.680680846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:25:51.683966233Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EB30511843, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:25:54.075275433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:25:54.821495099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:25:54.825199723Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EBEB88212D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:25:54.859451187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:25:54.862697463Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EBEDC7E4C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:25:56.547051673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:25:56.550487808Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EC52614497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=warn ts=2026-06-29T23:25:59.074704422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:02.266145334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:26:02.272205398Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EDA76BE955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:26:03.978061307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:26:03.985593929Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EE0D89E836, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:26:04.074405878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:04.163673109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:26:04.166948509Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EE185AA47F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:26:06.140889807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:26:06.145842537Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EE8E4D177F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:26:06.467024569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:26:06.473516105Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EEA1D66E8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:26:06.473535011Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:26:06.473558646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:26:06.478846565Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EEA229B668, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:26:09.075294821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:09.554564585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:26:09.561226372Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EF59E1F492, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:26:09.56124655Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:26:09.561269389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:26:09.567074406Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0EF5A3CB2C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:26:14.075172746Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:26:14.075205543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:26:14.075231029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:16.16564181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:26:16.171995847Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F0E3E8C487, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:26:17.002916923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:26:17.006153401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F115A1CD09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:26:17.701067388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:26:17.704485954Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F13F3FE7A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:26:19.075122382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:20.990192023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:26:20.99427955Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F20350D15B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:26:21.099264992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:26:21.103740698Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F209DCE965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:26:22.133866278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:26:22.139826188Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F2479F667C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:26:23.705095687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:26:23.70890197Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F2A5248ADE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:26:23.933424045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:26:23.944811706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F2B3313A85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:26:23.94483779Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:26:23.944867479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:26:23.955811886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F2B3DD7FF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:26:24.075090807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:26.117015413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:26:26.123075904Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F3350A896E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:26:26.501063542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:26:26.505094888Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F34BC95B87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:26:26.786957783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:26:26.790417222Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F35CD1A1F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:26:27.124488943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:26:27.129377214Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F370F26D7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:26:27.129392345Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:26:27.129413867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:26:27.132903544Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F37135DD38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:26:29.075263685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:29.559399476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:26:29.562817431Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F40210DFAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:26:30.724852954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:26:30.728194786Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F44787A30C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:26:31.197728164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:26:31.201098305Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F463B6E080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:26:31.949176799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:26:31.952548739Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F490811E6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:26:33.036491882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:26:33.040419215Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F4D1570DEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:26:34.074357052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:35.501835985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:26:35.507591335Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F5646874CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:26:36.42489373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:26:36.433516146Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F59B97F257, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:26:39.057685182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:26:39.06345177Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F63859551F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:26:39.074512529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:41.348238219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:26:41.354208091Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F6C0E507BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:26:44.056552636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:26:44.074454819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:44.074449539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:26:44.074488866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:26:44.609742697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:26:44.617823823Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F7835404AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:26:45.545079935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e336d5a7b81a6a0f immediate=false num_chunks=1 total_comp="3.2 kB" avg_comp="3.2 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:26:45.549232473Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F7BAEE4761, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dora-metrics\"}"
level=error ts=2026-06-29T23:26:45.549268184Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:26:45.54930936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="314 kB" avg_comp="314 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" max_age=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:26:45.557616954Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F7BB6F675A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:26:46.831428737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:26:46.837842109Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F807BC299B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:26:47.929854022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:26:47.933665227Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F8490DC801, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:26:49.074876196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:51.303833489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:26:51.307788518Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F91226B379, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:26:51.691839636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:26:51.695559867Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F92943520F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:26:54.003840426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:26:54.008960716Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F9B32B1EBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:26:54.074959264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:26:54.45348485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:26:54.459578713Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F9CE0585F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:26:54.920973977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:26:54.925255427Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0F9E9C7D293, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:26:58.323901574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:26:58.33039586Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FAB4BEEF08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:26:58.977001101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:26:58.983106749Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FADBA64028, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:26:59.075103819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:02.34147542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:27:02.345258173Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FBA40E9C6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:27:03.348092531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:27:03.354311292Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FBE02CD2AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:27:04.075082882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:06.429505701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:27:06.438219305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FC98033B3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:27:06.861565503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:27:06.867807315Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FCB19D1A61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:27:07.14852559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:27:07.151969099Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FCC28E77C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:27:09.074595192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:10.342869695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:27:10.346224979Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FD80F3B38C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:27:10.722243909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:27:10.725702412Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FD978FD61F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:27:10.725720051Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:27:10.725738468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:27:10.728763833Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FD97C03196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:27:14.074843467Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:27:14.074885229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:27:14.074948897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:15.999577169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:27:16.003564199Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FED2211BA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:27:16.025645243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:27:16.029076254Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FED3AAE1A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:27:19.075339954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:19.834198746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:27:19.837663704Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FFB6AD4459, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:27:20.213190873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:27:20.21654507Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FFCD431C0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:27:21.050090189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:27:21.053498416Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB0FFFF262742, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:27:21.500750367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:27:21.505468221Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1001A05FCA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:27:24.074975446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:24.941771711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:27:24.944972244Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB100E71BA577, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:27:26.590747092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:27:26.594873525Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB101496F4FCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:27:26.802410789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:27:26.809456524Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB101563BEA72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:27:27.378457545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:27:27.382437707Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10178606DDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:27:29.074589455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:29.898151712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:27:29.904150796Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1020EB07782, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:27:29.904173753Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:27:29.904199328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:27:29.906983396Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1020EDF2D7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:27:30.555848102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:27:30.561916105Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10235E675EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:27:31.041074705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:27:31.047069119Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10252D19636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:27:33.689606941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:27:33.693168568Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB102F0881AF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:27:34.074952674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:35.543592481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:27:35.546839774Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1035F0557E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:27:36.798369758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:27:36.802963139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB103A9E00ADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:27:39.074659517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:40.713408119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:27:40.720966909Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1049367C4F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:27:44.055621141Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:27:44.074463771Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:27:44.074491995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:27:44.074500611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:44.36606589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:27:44.369957819Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1056CEB1BB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:27:45.584187608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:27:45.591027587Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB105B5AE70D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:27:45.848811652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:27:45.852298895Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB105C5462601, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:27:46.975178098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:27:46.980462504Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB106088225DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:27:47.269369613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:27:47.275283637Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1061A17BC86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:27:48.743658303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:27:48.747291242Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10671D13B4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:27:48.747310854Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:27:48.747331031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:27:48.7506036Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10672035B3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:27:49.075269132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:49.088407241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:27:49.091728098Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB106865BC42E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:27:50.517640567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:27:50.520888005Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB106DB8BBAC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:27:52.087557622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:27:52.0909142Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB107391EDA87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:27:53.758352596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:27:53.761707584Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1079CB64E6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:27:54.07528266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:27:57.34232472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:27:57.352458799Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10872BC20C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:27:59.075172984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:00.535679042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:28:00.541431833Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10930D15EFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:28:04.075223023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:08.175023982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:28:08.178976832Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10AF808E65E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:28:09.074580241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:11.831606666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:28:11.835594755Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10BD2001760, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:28:13.133829646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:28:13.141642353Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C1FD8104F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:28:13.396386461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:28:13.399753312Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C2F3A8B57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:28:13.710747997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:28:13.714498638Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C41F96AF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:28:14.074334365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:28:14.074332805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:14.074368694Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:28:14.789510968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:28:14.792911124Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C8243E86A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:28:15.144950327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:28:15.148288021Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C9773A43D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:28:15.229408101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:28:15.23394922Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10C9C7EE965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:28:16.25705584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:28:16.26289216Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10CD9E37278, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:28:16.611678291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:28:16.617788171Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10CEF08E51B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:28:16.617796499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:28:16.620666116Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10CEF37C30C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:28:18.368499686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:28:18.37215231Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10D5799F77E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:28:19.075122893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:20.87282345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:28:20.877052616Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10DECE938D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:28:24.07461017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:24.338741542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:28:24.345035535Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10EBB9F52D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:28:24.518195479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:28:24.524486456Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10EC6505CF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:28:26.33760512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:28:26.341454628Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10F329806E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:28:28.63942361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:28:28.643355147Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10FBBD1BB5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:28:29.074505526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:29.507332641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:28:29.515140697Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10FEFC74E60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:28:29.515162214Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:28:29.515193688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:28:29.521692011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB10FF02E4272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:28:32.081975799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:28:32.08814186Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11089218543, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:28:32.761015538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:28:32.764307558Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB110B1736260, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:28:34.075178883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:34.935843177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:28:34.939116655Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1113314F16E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:28:36.643137083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:28:36.646617888Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11198D8C981, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:28:39.07517035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:39.691388673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:28:39.694806862Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1124E8A009D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:28:41.496561049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:28:41.499896532Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB112BA21FEF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:28:42.825441354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:28:42.829269592Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB113095929B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:28:44.056045397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:28:44.074393383Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:28:44.074420043Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:28:44.074436726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:45.627840085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:28:45.632817189Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB113B078D56E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:28:45.963286452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:28:45.969475149Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB113C4898953, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:28:49.074388486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:52.962004961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:28:52.96527037Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1156585AFEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:28:54.074445867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:28:54.713758804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:28:55.16768001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:28:55.173683932Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB115E9266A43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:28:55.521009838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:28:55.525067833Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB115FE11CEA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:28:56.445690533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11634F7C49F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:28:58.990389158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:28:58.997002739Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB116CD09B1D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:28:59.074736527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:01.643314241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:29:01.649565734Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1176B238D31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:29:03.592082707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:29:03.595257947Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB117DF1EE36F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:29:04.074383449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:05.02675361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:29:05.029939619Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11834A17F8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:29:06.412024948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:29:06.415127499Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB118873281FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:29:06.704055051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:29:06.707619878Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB118989E33DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:29:06.917526664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:29:06.92199991Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB118A5593BB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:29:09.075394659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:09.700833941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:29:09.704321468Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1194B3F77CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:29:10.392982027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:29:10.398743028Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB119749FFFB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:29:10.953251349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:29:10.960525234Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB119961EC2BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:29:13.850440761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:29:13.856365874Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11A42B97DAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:29:14.07439188Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:29:14.07443314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:29:14.074435949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:14.079741368Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-06-29T23:29:14.081801616Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007221033s
level=info ts=2026-06-29T23:29:15.424471501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:29:15.428214546Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11AA06A132C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:29:18.682431794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:29:18.688776323Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11B62C1348B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:29:19.075045301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:20.151211099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:29:20.154903538Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11BBA269631, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:29:21.936409381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:29:21.942400607Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11C24B0EC8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:29:21.942422348Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:29:21.94244777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:29:21.945275657Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11C24DF1E10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:29:22.289493013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:29:22.294397798Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11C39AC0A39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:29:24.075317329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:25.415862624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:29:25.42233746Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11CF417E51F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:29:25.422358015Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:29:25.422382371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:29:25.425833405Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11CF44CCD89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:29:27.044291139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:29:27.047744256Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11D54FD0ECD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:29:29.074563224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:30.82581616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:29:30.830996167Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11E367DBCC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:29:30.83101428Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:29:30.831032725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:29:30.834151395Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11E36AE6009, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:29:31.492176046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:29:31.495593121Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11E5E1A80F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:29:31.495621147Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:29:31.495654268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:29:31.498459861Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11E5E493FC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:29:32.356815001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:29:32.360271244Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11E91A3A971, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:29:34.075005042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:35.294976192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:29:35.298390019Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB11F40C57F0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:29:39.074916139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:39.225374224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:29:39.228936998Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1202B0AA11E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:29:40.469408797Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB12074FEAABE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:29:41.648561486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:29:41.652612828Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB120BB7B3882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:29:41.883365544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:29:41.891776109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB120C9C3356E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:29:42.81019313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:29:42.816376577Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12100E036E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:29:43.928834373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:29:43.932677127Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12143640AE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:29:44.055733455Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:29:44.074986223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:29:44.075019686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:29:44.075023248Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-06-29T23:29:44.075042792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:44.075157Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-06-29T23:29:44.075167069Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:29:44.075186254Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.858µs success=true
level=info ts=2026-06-29T23:29:46.000033258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:29:46.003400053Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB121BED67A39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:29:49.074706523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:49.375517297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:29:49.378929484Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1228809443C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:29:49.760988466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:29:49.764246459Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1229F0144CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:29:50.32173448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:29:50.324952012Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB122C06C824C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:29:50.968957655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:29:50.972755112Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB122E7042E8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:29:51.134309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:29:51.137672244Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB122F0DCEEAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:29:51.635938883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:29:51.639131341Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1230EC17BF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:29:52.363286637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:29:52.367817287Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1233A2F5912, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:29:54.075325002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:55.768016782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:29:55.77592993Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12405517040, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:29:56.732020033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:29:56.739309016Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1243EB076AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:29:58.049827215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:29:58.053329031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1248D129D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:29:59.013061279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:29:59.016513491Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB124C67C646E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:29:59.01653048Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:29:59.016549885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:29:59.019258257Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB124C6A87B1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:29:59.07440374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:29:59.558887456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:29:59.564514409Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB124E72458A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:30:04.074387651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:04.445225535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:30:04.451594521Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1260A6F2A1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:30:04.451640039Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:04.451662321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:30:04.454520846Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1260A9EA681, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:30:05.563817687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:30:05.567466328Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1264CEEFBFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:30:05.567486391Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:05.567510832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:30:05.570406308Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1264D227BC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:30:05.788040042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:30:05.793171841Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1265A671C09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:30:08.729286336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:30:08.735540661Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12709C31843, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:30:09.075302532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:10.862178673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:30:10.865824814Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12788C0B4CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:30:13.563678114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:30:13.56750823Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12829CA54C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:30:14.075227969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:30:14.075264942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:30:14.075265563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:14.309663285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:30:14.31562024Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1285661139D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:30:16.043960845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:30:16.047358996Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB128BD99FE57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:30:16.087670944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:30:16.090993026Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB128C032F51F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:30:16.166211226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:30:16.169426309Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB128C4E156A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:30:16.258684408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:30:16.261947982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB128CA6432F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:30:17.319181238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:30:17.322993887Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12909A1990D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:30:17.365105987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:30:17.371140928Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1290C80DE99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:30:17.371156235Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:17.371175568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="149 kB" avg_uncomp="149 kB" max_age=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:30:17.374229848Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1290CB1B0F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:30:17.947491873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:30:17.951238785Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1292F0F8B01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:30:19.074995006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:23.375437804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:30:23.378919032Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12A7296241A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:30:24.07444698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:25.797225304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:30:25.801244149Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12B02F30D66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:30:27.45366734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:30:27.457038903Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12B65AAF3DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:30:29.074610484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:29.674729842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:30:29.678116912Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12BEA0DCDE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:30:29.678133086Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:29.678151234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:30:29.68095967Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12BEA3B46F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:30:31.542300207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:30:31.54571678Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12C595EBA83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:30:31.701260514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:30:31.704782596Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12C62D96BAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:30:31.704800882Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:31.704823542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:30:31.707587989Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12C63073531, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:30:31.930224069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:30:31.933696383Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12C707E3208, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:30:33.590190785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:30:33.594452402Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12CD37CA434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:30:34.074611869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:35.480272097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:30:35.487436946Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12D4451AF00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:30:35.925638489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:30:35.929071515Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12D5EA00968, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:30:37.05091052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:30:37.05861295Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DA1F838B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:30:37.408467281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:30:37.414686589Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DB7310481, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:30:37.599192566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:30:37.603023194Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DC2663893, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:30:37.603041993Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:37.603063043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:30:37.606523095Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DC29B27EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:30:37.752054697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:30:37.755949606Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DCB879CE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:30:37.755968684Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:37.903302689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:30:37.909436691Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12DD4AF327C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:30:39.074653949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:43.921064557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:30:43.927255607Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12F3B5E7071, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:30:44.055616123Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:30:44.074709915Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:30:44.074739491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:30:44.074742404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:44.124111978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:30:44.127651125Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB12F474EA801, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:30:48.470291175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:30:48.474903796Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1304A5E6090, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:30:49.07522626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:49.168876073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:30:49.173130159Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB130740C5DDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:30:51.028867615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:30:51.034922316Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB130E30060D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:30:51.786142481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:30:51.789790344Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB131100321FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:30:52.903968674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:30:52.907820631Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13152A1D00B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:30:54.074893649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:55.781674844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:30:55.785140103Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB131FE26E1D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:30:55.820168341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:30:55.823395121Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB132006F85B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:30:55.823408745Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:30:55.823427335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:30:55.826171914Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB132009BD5DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:30:57.657780703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:30:57.662864257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1326E1248CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:30:59.075345133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:30:59.50707398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:30:59.515690712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB132DC815036, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:31:02.601613425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:31:02.607743077Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13394CF45A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:31:03.546216994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:31:03.550204546Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB133CCFD1FA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:31:04.074939057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:04.945276982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:31:04.95139699Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB134208133C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:31:05.811037771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:31:05.814777258Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13453F2020A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:31:07.010222589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:31:07.013532391Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1349B6B20D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:31:07.013549393Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:31:07.013567772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:31:07.016324143Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1349B98713F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:31:09.07466947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:09.194990196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:31:09.198332764Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1351DA4C6ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:31:09.877574724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:31:09.881006651Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1354654A112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:31:12.824201792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:31:12.828038693Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB135F5FE3694, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:31:13.449740718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:31:13.456505817Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1361B73FE14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:31:13.826461504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:31:13.83220932Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13631D7ADE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:31:14.074619203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:31:14.074648926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:14.074672461Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:31:14.15919733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:31:14.162379204Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB136458706AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:31:14.162396461Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:31:14.162416001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:31:14.165080734Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13645B18BBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:31:15.564427307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:31:15.567810279Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB136994BD34D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:31:16.566660848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:31:16.570871953Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB136D5149093, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:31:17.543873452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:31:17.549892608Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1370F6F54D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-29T23:31:17.549912366Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:31:17.549935224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:31:17.552743169Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1370F9C4515, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:31:17.878421547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:31:17.881788648Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13723382B3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:31:18.440398444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:31:18.444115093Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13744B80E97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:31:19.074366691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:19.895592076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:31:19.898860851Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1379B72A95A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:31:19.898881268Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:31:19.898901948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:31:19.901654887Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1379B9DC327, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:31:20.035145476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:31:20.038846652Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB137A3C5949A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:31:20.783609059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:31:20.786926824Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB137D0618EF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:31:24.074636073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:24.744545763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:31:24.747935481Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB138BC78570B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:31:26.351829331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:31:26.359665175Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1391C8A18D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:31:29.075111287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:32.37725669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:31:32.385009665Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13A83AC5923, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:31:33.894045527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:31:33.897500711Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13ADDD42238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:31:34.075074733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:34.219397499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:31:34.222991178Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13AF1398978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:31:37.591029446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:31:37.594847606Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13BBA329766, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:31:38.591764578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:31:38.595185723Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13BF5D422B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:31:39.075373745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:40.568774605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:31:40.572277095Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13C6BAD7620, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:31:41.02660522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:31:41.031378286Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13C86F8D15D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:31:42.490616859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:31:42.494135786Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13CDE39EE48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:31:43.564493426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:31:43.568236456Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13D1E3E2281, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:31:44.056119252Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:31:44.074720928Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:31:44.074751158Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:31:44.074774076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:47.876376873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:31:47.879834405Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13E1F3CFFA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:31:48.666262466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:31:48.670198266Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13E4E5A2DA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:31:49.07506366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:49.389231119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:31:49.39600235Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13E7999BDF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:31:49.960666973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:31:49.96387868Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13E9B752043, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:31:49.972916002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:31:49.978009253Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13E9C4E39F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:31:53.561411461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:31:53.568086173Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13F7249E663, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:31:54.07237917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:31:54.07512759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:31:54.080178711Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13F90CEDB6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:31:54.473552736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:31:54.479707609Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13FA89F399A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:31:55.372428124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:31:55.375694804Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB13FDE07ED7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:31:56.646437488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:31:56.649955539Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14029F94D85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:31:59.074551277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:31:59.599325868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:31:59.602656572Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB140D9FA67AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:32:02.633420282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:32:02.636860676Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1418ED4B04C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:32:04.075039965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:04.243550607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:32:04.246887113Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB141EECB7112, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:32:04.850597172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:32:04.85444238Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14212FC7E28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:32:05.843185179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:32:05.847210187Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1424E2EB9CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:32:06.94239862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:32:06.949462289Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1428FE0EBA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:32:09.074574319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:13.793554556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:32:13.796963136Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1442804972E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:32:13.869320557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:32:13.873578223Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1442C95DF5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:32:13.873597559Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:32:13.873639362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:32:13.879649065Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1442CF47A01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:32:14.074669203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:32:14.07468661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:14.074699846Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:32:14.712132251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:32:14.717942286Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1445EE6EA00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:32:19.074791046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:20.137158896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:32:20.140450532Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB145A2209688, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:32:20.581667095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:32:20.584957069Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB145BC9E7B09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:32:20.762660414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:32:20.76579965Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB145C766BCD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:32:22.91315146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:32:22.921724958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14647E68F01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:32:23.582001607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:32:23.588479906Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1466FA15D04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:32:24.074662984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:25.606894201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:32:25.610072708Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB146E82481C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:32:26.145650729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:32:26.149715312Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB147084DCCB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:32:29.074704495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:30.602848124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:32:30.609203856Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB148121C2BBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:32:34.043130854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:32:34.046550083Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB148DEFD01C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:32:34.074371333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:36.237564045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:32:36.242617765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14961E1BBB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:32:37.032737605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:32:37.039277469Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB149915F405C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:32:37.039297561Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:32:37.039320238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:32:37.044907562Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14991B76B5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:32:38.328081655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:32:38.334287489Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB149DE911E4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:32:38.561032939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:32:38.567185159Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB149EC6D71CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:32:38.865957878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:32:38.871353555Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB149FE924606, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:32:38.871373348Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:32:38.871398661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:32:38.878995797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB149FF0831BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:32:39.051395938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:32:39.058124558Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14A09A38389, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:32:39.07477689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:41.034109668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:32:41.037331677Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14A7FADC610, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:32:42.677051275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:32:42.680576407Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14AE19E0199, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:32:44.055572729Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:32:44.075132232Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:32:44.075163721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:32:44.075161138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:44.960769248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:32:44.964437395Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14B69BFBF2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:32:47.039472733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:32:47.042994194Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14BE5A3B5B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:32:48.330681541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:32:48.334430465Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14C329DEEE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:32:49.075371077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:50.184636592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:32:50.191217455Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14CA149E9EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:32:50.605479812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:32:50.609199404Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14CBA2FC871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:32:51.398895698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:32:51.403077928Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14CE9847489, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:32:53.414826595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:32:53.420978305Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14D61CB51A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:32:54.07461624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:32:55.539096034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:32:55.542500151Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14DE03EBFB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:32:55.849318666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:32:55.855074514Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14DF2E2C136, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:32:58.883573669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:32:58.889875609Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14EA7C4D392, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:32:59.074923479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:01.103671007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:33:01.109407039Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14F2C10D948, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:33:02.605772083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:33:02.609748373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB14F857942F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:33:04.07439023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:06.337783143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:33:06.341032052Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15063E5ECDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:33:09.075350651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:10.991616484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:33:10.994906518Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB151794A9E0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:33:11.502363056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:33:11.505879923Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15197BF1BB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:33:12.464435792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:33:12.47674475Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB151D19C31B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:33:12.476766425Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:33:12.476792517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:33:12.487172648Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB151D23CAC1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:33:12.764910358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:33:12.770879867Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB151E324DAEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:33:13.125879688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:33:13.129601892Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB151F880E24F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:33:14.074720984Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:33:14.074750085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:14.074757999Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:33:14.567289203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:33:14.570734227Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1524E6BDD29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:33:18.35226778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:33:18.355836187Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15330082D9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:33:19.074759129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:19.580104732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:33:19.583605195Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB153793497AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:33:23.164672142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:33:23.168609549Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1544EDF3381, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:33:23.302960769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:33:23.306238786Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB154571A0602, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:33:24.07454716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:25.620215596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:33:25.629067366Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB154E18C89CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:33:26.253828847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:33:26.261396363Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB155073D22DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:33:26.695521834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:33:26.700835393Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB155216E9C75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:33:29.075265162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:33.598015433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:33:33.604821759Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB156BCF0D63E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:33:34.035068096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:33:34.041075584Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB156D6F258BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:33:34.041096726Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:33:34.041118974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:33:34.043867614Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB156D71FBD0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:33:34.074645197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:35.01217465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:33:35.018092791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB157112E8569, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:33:35.470653325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:33:35.476137648Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1572C7ADD1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:33:37.64732642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:33:37.651948994Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB157AE1B2DFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:33:37.651967569Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:33:37.651989081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:33:37.65532714Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB157AE5BA414, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:33:39.075113992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:39.139267764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:33:39.142575796Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1580704EECA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:33:39.19630418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:33:39.19950759Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1580A698137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:33:40.610468565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:33:40.614180805Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1585EB81A7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:33:43.195984056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:33:43.199123092Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB158F8CFCBFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:33:44.055902129Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:33:44.074723943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:33:44.074752522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:33:44.074808693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:45.653054982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:33:45.656937285Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1598B4E5F8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:33:45.656958047Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:33:45.656980671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:33:45.663491447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1598BB234D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:33:46.950058559Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB159D863B441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:33:48.632401272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:33:48.638468372Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15A3D04808E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:33:49.074790669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:49.300450505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:33:49.30514855Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15A64B0828E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:33:50.4429623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:33:50.446226454Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15AA8C61EE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:33:50.707090954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:33:50.71126802Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15AB89237B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:33:54.076393755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:33:54.759022686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:33:54.765435094Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15BAA338384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:33:55.025127667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:33:55.028340282Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15BB9E2A5F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:33:56.173831698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:33:56.177140864Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15BFE5AF61B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:33:57.652237502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:33:57.656091152Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15C5682EDB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:33:59.074520398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:00.74467148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:34:00.751162519Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15D0EFD3EF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:34:03.577286772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:34:03.581247349Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15DB7ACC63A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:34:04.074507361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:05.43313821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:34:05.443784911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15E26B0C414, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:34:06.191055871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:34:06.197309019Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15E539A9AAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:34:09.075330331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:09.413146181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:34:09.416730469Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15F137E222F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:34:11.417660356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:34:11.421123943Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15F8AF796BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:34:11.574679465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:34:11.579651414Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15F9456C94F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:34:11.876439808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:34:11.880286727Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15FA6504DBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:34:12.820933005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:34:12.824292864Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB15FDE9ABEBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:34:14.074947942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:34:14.074974038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:14.074987805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:34:14.080954415Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-06-29T23:34:14.082956374Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007627185s
level=info ts=2026-06-29T23:34:14.369141836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:14.372793027Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1603AE512A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:14.372809863Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:34:14.372830338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:14.375706304Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1603B148BAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:34:15.038424926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:34:15.042268076Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16062C8137C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:34:17.905755018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:34:17.909687574Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1610DB76BED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:34:19.074606318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:20.016913871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:34:20.024655299Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1618BC64C6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:34:20.332231257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:34:20.335572163Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1619E4FD870, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:34:24.075013568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:24.251545819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:34:24.256898435Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16288096B54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:34:24.835058552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:34:24.8413298Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB162AADEAC21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:34:25.315529469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:34:25.319303551Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB162C75DACB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:34:27.63606982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:34:27.642199903Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16351D2A16A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:34:29.074793743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:29.458323124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:29.461617654Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB163BE433F1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:34:33.463566276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:34:33.466945364Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB164ACFF7B09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:34:34.068205955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:34:34.072343479Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB164D116C9CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:34:34.074390443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:34.703526291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:34:34.711652388Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB164F730FF81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:34:34.999403394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:34:35.005188081Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16508B04242, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:34:36.24415829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:34:36.247427298Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16552BC34B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:34:39.075237953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:39.928609182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:34:39.932514744Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1662E5C010A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:34:39.932533342Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:34:39.932552254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:34:39.935862928Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1662E9103BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:34:40.793262738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:34:40.796560897Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16661E1E035, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:34:44.0564163Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:34:44.07419359Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:34:44.074317899Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-06-29T23:34:44.074327984Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:34:44.074351397Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.209µs success=true
level=warn ts=2026-06-29T23:34:44.074349094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:44.0744209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:34:44.074445803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:34:46.190340374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:34:46.194369029Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB167A39933A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:34:49.075324599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:49.087493347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:34:49.091182689Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16850484A9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:34:52.482168148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:34:52.489921342Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1691AD6AE62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:34:54.074466226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:55.542345912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:34:55.545984428Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB169D1019BAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:34:56.754515268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:34:56.757976411Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16A194108AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:34:57.932491161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:34:57.936470766Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16A5F7A6C8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:34:58.75620324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:34:59.075144104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:34:59.662348048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:59.668478514Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16AC6BC73DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:34:59.68837652Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16AC7EC9D3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:34:59.700445455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:34:59.705956255Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16AC8F83F30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:35:00.906583681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:35:00.910368506Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16B10C1F9CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:35:01.845160723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:35:01.851354986Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16B48D84006, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:35:02.004690342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:35:02.008109418Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16B522EACAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:35:02.865379261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:35:02.86889523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16B857EF812, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:35:04.074885195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:06.120059797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:35:06.123409767Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16C477A32BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:35:07.776670253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:35:07.780031134Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16CAA3742E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:35:07.801059822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:35:07.804238147Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16CABAB4342, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:35:07.829267535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:35:07.832338814Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16CAD587115, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:35:07.881672552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:35:07.885376087Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16CB07BEF2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:35:09.07459842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:09.289796602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:35:09.293669927Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16D046C0FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:35:14.074758344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:35:14.074792585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:35:14.074783914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:17.76856856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:35:17.772085826Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16EFDCB6B65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:35:17.999643892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:35:18.002927045Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16F0B8DC581, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:35:19.074661348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:22.051278135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:35:22.056242872Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16FFD26A1DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:35:22.056262276Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:35:22.056285298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:35:22.064838476Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB16FFDA8B2BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:35:22.395679224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:35:22.402339933Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17011C5249F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:35:23.228665861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:35:23.232722092Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1704345D64C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:35:24.075291425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:24.307068097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:35:24.313763514Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17083B576B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:35:26.181678975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:35:26.189783074Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB170F386E18E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:35:28.018406469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:35:28.024514396Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17160E2E4B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:35:29.07496286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:32.121043993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:35:32.126071378Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB172555BC1E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:35:34.074375178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:35:39.074914328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:40.472663503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:35:40.47985552Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1744746DC45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:35:40.612070236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:35:40.615441678Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1744F5CCADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:35:42.177661137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:35:42.181954297Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB174ACB40D89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:35:42.181973642Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:35:42.181995127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:35:42.185794997Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB174ACF13DA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:35:43.400979984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:35:43.404522258Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB174F5997F10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:35:43.404540691Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:35:43.404559378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:35:43.407380749Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB174F5C8CD0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:35:43.719294204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:35:43.722909393Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1750893BF6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:35:44.05557908Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:35:44.074878542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:35:44.074906795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:35:44.074909982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:44.273338109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:35:44.277401453Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB175299B329E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:35:45.350554742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:35:45.354399762Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17569D35931, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:35:48.19705354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:35:48.20325788Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17613A0D4E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:35:48.989478909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:35:48.992858387Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17642B13A92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:35:49.07494885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:49.523057526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:35:49.526386164Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17662801651, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:35:50.866810277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:35:50.870212622Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176B2988B56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:35:50.870230531Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:35:50.870250357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:35:50.873051071Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176B2C602BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:35:50.879124735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:35:50.882008771Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176B34E2F29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:35:51.183920278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:35:51.187208939Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176C57E7BFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:35:51.18722785Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:35:51.187248851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:35:51.190089655Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176C5AB7ECC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:35:52.101619022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:35:52.105207864Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB176FC350CDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:35:52.926656489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:35:52.934500196Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1772DA44E95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:35:54.075183803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:35:55.197202175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:35:55.203524871Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB177B4E0ACDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:35:55.679821373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:35:55.683759705Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB177D17A8965, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:35:56.619062645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:35:56.623656963Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1780974D9A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:35:56.883520446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:35:56.88716052Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB178193BC473, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:35:57.292419562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:35:57.296398728Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB178319F3DF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:35:59.074866748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:02.205297764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:36:02.211462816Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17956965233, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:36:04.074408981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:04.153825338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:36:04.157769341Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB179CA99133E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:36:06.710577209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:36:06.716960116Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17A63211F12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:36:07.684742054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:36:07.688032165Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17A9D04824F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:36:09.075096473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:09.184416055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:36:09.189383177Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17AF681D8AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:36:10.380602974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:36:10.388984468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17B3E013F7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:36:10.603533027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:36:10.609048854Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17B4B1FA3FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:36:12.508827778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:36:12.513112616Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17BBC9C58DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:36:14.074585066Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:36:14.074592331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:14.074616715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:36:15.255822218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:36:15.261643699Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17C60703380, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:36:17.260027182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:36:17.263416303Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17CD7C18F1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:36:19.075424276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:20.617863987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:36:20.621738377Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17D9FE9D23D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:36:21.065270524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:36:21.069063356Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17DBA92B672, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:36:23.698378884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:36:23.701690936Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17E57806808, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:36:24.074501861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:26.83380416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:36:26.837612236Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17F1267650D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:36:28.468578944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:36:28.4717978Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17F73D444FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:36:29.07509522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:30.040559435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:36:30.043884452Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17FD18721E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:36:30.076600017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:36:30.079771422Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB17FD3ABC47C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:36:31.011758801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:36:31.015383832Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1800B6FEE37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:36:31.629270772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:36:31.636778375Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1803077DE81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:36:32.116117942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:36:32.11949639Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1804D3E2DDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:36:32.522559885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:36:32.525884966Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1806576A0AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:36:33.162755145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:36:33.166113828Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1808BA052D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:36:33.166130737Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:36:33.166150841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:36:33.168917417Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1808BCD95B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:36:33.769656285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="98 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:36:33.779772594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB180B032B5A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:36:34.074610808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:36:39.074829567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:43.506829267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:36:43.514761748Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB182F472DEBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:36:44.055825028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:36:44.074736288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:44.074745688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:36:44.074772253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:36:45.646677129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:36:45.652446966Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18373CCD1C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:36:45.911974627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:36:45.915169124Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1838387B7DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:36:45.915191343Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:36:45.915210485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:36:45.918676184Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18383B9D94A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:36:46.127356922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:36:46.13111084Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18390628A5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:36:46.131129325Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:36:46.21527779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:36:46.218706087Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB183959EE8D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:36:48.183139974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:36:48.186501111Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1840AE9895D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:36:49.07536914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:51.945674282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:36:51.948961928Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB184EB2C182F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:36:53.246504361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:36:53.252007118Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18538D6202C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:36:53.672276875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:36:53.678999632Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18552482221, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:36:54.075167835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:54.290069657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:36:54.297191733Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18577225CB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:36:54.2972149Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:36:54.29723962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:36:54.303412274Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1857784772E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:36:56.310072073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:36:56.316223804Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB185EF7B3BC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:36:56.492774789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:36:56.496619284Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB185FA3650FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:36:59.074503531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:36:59.080662085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:36:59.084175432Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18694740C2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:37:01.72484128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:37:01.728369685Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB187321169A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:37:01.728386943Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:01.728405807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:37:01.731204752Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB187324002CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:37:03.090172607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:37:03.094133841Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18783729F20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:37:04.074744297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:08.718828295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:37:08.723996398Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB188D30A48AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:37:08.912821594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:37:08.921286136Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB188DECD7D01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:37:09.07460668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:09.444662397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:37:09.450854022Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB188FE5E7F2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:37:09.921340307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:37:09.926989652Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1891ABEBC30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:37:11.816594724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:37:11.820243035Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1898B9478AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:37:11.948225585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:37:11.95146694Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB189936A4F9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:37:12.303410641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:37:12.306744564Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB189A8982AF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:37:12.306763785Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:12.306781055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:37:12.309587383Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB189A8C54A72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:37:14.075156418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:14.075161642Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:37:14.075193751Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:37:14.364679586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:37:14.368108976Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18A23746CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:37:14.368126148Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:14.368145016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:37:14.37091597Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18A23A1F684, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:37:14.754230397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:37:14.759255433Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18A3AC3CEDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:37:16.189506727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:37:16.865557223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18AB851DCAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:37:16.876545851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:37:16.881742738Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18AB94889FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:37:17.205550003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:37:17.208776035Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18ACCC6AEEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:37:17.733472468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:37:17.736745252Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18AEC3EA6CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:37:19.074902284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:19.54899341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:37:19.552182425Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18B58750867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:37:20.103227639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:37:20.106860329Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18B79845AB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:37:20.983734661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:37:20.98942455Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18BAE1E3A40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:37:21.571205446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:37:21.574461579Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18BD0FDAF77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:37:24.074383987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:24.703825528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:37:24.707654826Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18C8BB9BB96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:37:24.707673647Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:24.707695232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:37:24.711091404Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18C8BEED401, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:37:25.233388441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:37:25.236756512Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18CAB477B91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:37:27.483432282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:37:27.486696447Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18D3163354D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:37:27.486715565Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:27.486734301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:37:27.489515576Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18D31913682, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:37:27.953735543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:37:27.957440483Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18D4D6D749A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:37:29.07537658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:29.345148027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:37:29.34853563Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18DA05BE1F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:37:33.05167005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:37:33.056405836Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18E7D4E489B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:37:34.075305143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:35.080172604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:37:35.085683072Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18EF652A7C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:37:37.112293705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:37:37.118212795Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18F6F78B178, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:37:38.049237133Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB18FA6F941C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:37:38.788266944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:37:38.79210847Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB18FD3394F82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:37:39.075078558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:43.871119897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:37:43.874764015Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1910230E5E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:37:44.056179076Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:37:44.075127162Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:37:44.075129914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:44.075165153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:37:47.016601971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:37:47.020660332Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB191BDB34F83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:37:47.773229759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:37:47.778841806Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB191EAE48650, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:37:47.778861601Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:47.778882971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:37:47.781803003Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB191EB1421BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:37:49.075175937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:51.638134656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:37:51.64193088Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB192D1264910, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:37:52.226605725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:37:52.233988624Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB192F471C281, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:37:54.074840143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:54.702829058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:37:54.708899144Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19387F54A30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:37:54.708917984Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:37:54.70893675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:37:54.711710407Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1938821EAB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:37:56.199975114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:37:56.203385824Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB193E109BA34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:37:56.803485594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:37:56.807111677Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19405029830, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:37:57.088069777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:37:57.091327318Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19415F69E38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:37:58.167533684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:37:58.17092933Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1945650321B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:37:59.074584496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:37:59.576670934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:37:59.580796767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB194AA58A832, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:38:00.327646661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:38:00.33417773Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB194D73B4EBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:38:00.98002229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:38:00.983281675Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB194FDF08F58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:38:01.947084329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:38:01.950597826Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1953796D272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:38:02.638456091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:38:02.642004975Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19560CDB3EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:38:04.075363387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:05.007150747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:38:05.011019353Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB195EDFD7376, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:38:05.185938402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:38:05.189204157Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB195F8A28181, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:38:05.189220776Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:38:05.189239502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:38:05.191974298Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB195F8CF2CEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:38:05.291453915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:38:05.295032086Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB195FEF1AC97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-29T23:38:05.295045316Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:38:05.4895836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:38:05.492860218Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1960ABA87B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:38:06.699447199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:38:06.708522855Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19653313875, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:38:06.891144108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:38:06.897294838Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1965E72CFBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:38:07.095071587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:38:07.100863865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1966A9407E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:38:09.074718143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:10.645685488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:38:10.649131418Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1973E121280, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:38:11.773013028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:38:11.776391531Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19781436CAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:38:14.074739417Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:38:14.074773278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:14.0747844Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:38:14.636747708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:38:14.640250728Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1982BF49D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:38:15.244152656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:38:15.247298998Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19850250926, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:38:15.494431489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:38:15.498102246Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1985F13C441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:38:18.297295552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:38:18.302658819Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB199062C3B51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:38:19.075335442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:20.517742704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:38:20.521121101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1998A7BA6A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:38:20.891828886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:38:20.897309792Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB199A0E9BA04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:38:22.21507443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:38:22.222639357Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB199EFE6672D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-06-29T23:38:24.074402612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:24.089468979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:38:24.092786494Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19A5F5E4B0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:38:26.864958122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:38:26.868315951Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19B04CD4B86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:38:29.074603182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:31.394643831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:38:31.398578116Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19C12D48B9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:38:34.075071138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:36.827621588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:38:36.83366708Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19D56C9B139, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:38:39.075301256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:41.620657053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:38:41.624408617Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19E745531C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:38:44.055646891Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:38:44.075074235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:44.075101217Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:38:44.075128749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:38:45.389981647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:38:45.396761891Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19F552B38F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:38:45.988274676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:38:45.993482417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB19F78C13805, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:38:48.537677947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:38:48.544411297Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A010CE0EC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:38:48.544430767Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:38:48.544450714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:38:48.550393386Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A011299806, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:38:48.734863921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:38:48.740771246Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A01C816313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:38:49.074829553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:49.748301639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:38:49.751808434Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A058C430EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:38:49.876697966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:38:49.880090437Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A0606A444A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:38:50.691962979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:38:50.695338203Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A091010E22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:38:51.137433473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:38:51.140654826Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A0AB8CD94E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:38:51.177816437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:38:51.181423918Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A0ADF6776C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:38:51.975597235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:38:51.979029361Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A0DD84F5D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:38:53.636056113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:38:53.639850656Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A1407E783F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:38:54.075020943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:55.207108231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:38:55.210520633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A19E217EAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:38:57.103393825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:38:57.106885556Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A20F299E5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:38:59.075118534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:38:59.9460363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:38:59.949334353Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A2B896A35F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:39:00.997012501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:39:01.000364513Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A2F73B8857, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:39:01.328413511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:39:01.333791397Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A30B1B5C34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:39:01.333812955Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:39:01.333836323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:39:01.342857819Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A30BA6D535, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:39:02.650972317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:39:02.657618873Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A35A03BC49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:39:02.827935156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:39:02.834719234Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A36487362B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:39:02.915026636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:39:02.922982397Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A369D4A7D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:39:03.535659089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:39:03.541405217Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A38EAF28FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:39:04.074898806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:04.541399447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:39:04.545031806Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A3CA83BC11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:39:04.545051015Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:39:04.545073677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:39:04.551521428Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A3CAE78DBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:39:05.262409017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:39:05.268123469Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A3F59C4F3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:39:07.647768604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:39:07.651171318Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A483A74870, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:39:09.07500868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:11.390025715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:39:11.393481986Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A562B6FE63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:39:12.529397085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:39:12.532728867Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A5A69D5433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:39:14.07439026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:39:14.074389822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:14.074423304Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:39:14.082238297Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-06-29T23:39:14.084332494Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009826813s
level=info ts=2026-06-29T23:39:15.274679322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:39:15.27866733Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A64A48C12D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:39:17.706413465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:39:17.714255327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A6DB76DB75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:39:19.075029804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:19.200317956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:39:19.206739885Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A7346A4804, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:39:24.074770891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:25.981255941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:39:25.98823018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A8C8A1DA5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:39:28.64656176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:39:28.652656254Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1A9676FD80B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:39:29.074377639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:33.203844033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:39:33.20723466Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AA76EAC1C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:39:33.427761114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:39:33.431118293Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AA8443A59F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:39:34.074682731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:34.436511743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:39:34.439799713Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AAC0615FE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:39:36.308194914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:39:36.312278309Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AB2FFD9261, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:39:38.777022012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:39:38.783294256Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ABC345D09B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:39:39.074964653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:39.120126555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:39:39.123440406Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ABD78C9123, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:39:40.549404284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:39:40.554309908Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AC2CD6957C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:39:41.488897135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:39:41.495377391Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AC64E959A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:39:41.694442631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:39:41.6980989Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AC7102CF11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:39:43.76344346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:39:43.769395033Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ACEC78926F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:39:44.055651806Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:39:44.074739277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:44.074745374Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:39:44.074766198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:39:44.074781936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:39:44.074952661Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-06-29T23:39:44.074965732Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:39:44.07499651Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=25.46µs success=true
level=info ts=2026-06-29T23:39:46.396635993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:39:46.400394029Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AD8946E43A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:39:46.586888122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:39:46.59478684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AD94DFCF3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:39:46.594807823Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:39:46.594834282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:39:46.606613616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AD95946F2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:39:46.632732252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:39:46.639091067Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AD9784E060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:39:48.714666828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:39:48.718087188Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AE136E86A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:39:48.718106416Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:39:48.718133774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:39:48.720892373Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AE139C0055, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:39:49.07523394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:49.421390585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:39:49.427121167Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AE3DB25E11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:39:49.64462145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:39:49.650430813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AE4AFF2B47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:39:50.393331998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:39:50.396905412Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AE777DFB94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:39:51.74040045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:39:51.744937815Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AEC7CB9769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:39:51.744957063Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:39:51.744979267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:39:51.748242616Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AEC8094137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:39:54.074819255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:39:55.952544913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:39:55.956187714Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AFC2DCD202, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:39:56.969522968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:39:56.97625697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1AFFFA37358, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:39:58.827689116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:39:58.831214137Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B06E374014, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:39:59.075087007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:01.429298685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:40:01.432779721Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B109492748, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:40:01.855277893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:40:01.858621625Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B122AC9CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:40:02.041060581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:40:02.044410695Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B12DBDF54B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:40:03.685826659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:40:03.689223818Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B18FC83157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:40:04.075207483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:04.375372585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:40:04.383281068Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B1B927406B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:40:06.190128527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:40:06.197080341Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B22536DCB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:40:08.636743263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:40:08.639984183Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B2B6DF91A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:40:09.074669961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:40:14.074829435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:14.074847226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:40:14.074880721Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:40:17.315536208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:40:17.319287303Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B4BC331B00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:40:17.319307642Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:40:17.319335184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:40:17.327083603Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B4BCAAE9A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:40:17.603772767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:40:17.609774029Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B4CD835BC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:40:19.075008806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:19.438538201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="775 B" avg_comp="775 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:40:19.44179199Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B53AB61A5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:40:22.415878953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:40:22.419322405Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B5EC2ED0E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:40:24.074611072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:25.702470295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:40:25.705777906Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B6B012DB80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:40:26.895887642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:40:26.899126788Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B6F734F87D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:40:27.718590011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:40:27.721900911Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B7283FF1D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:40:29.074743525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:29.879308216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:40:29.883010201Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B7A90EC134, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:40:30.263031697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:40:30.270294187Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B7C023FDB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:40:32.368902023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:40:32.375562287Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B83D9E4122, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:40:32.85374124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:40:32.860197235Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B85A7CA645, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:40:33.422513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:40:33.431651497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B87C9026DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:40:34.075289069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:37.122676557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:40:37.129066395Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B958F17633, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:40:37.386838396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:40:37.390250643Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B968869F8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:40:37.445300907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:40:37.450648576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B96C1F15C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:40:37.532840502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:40:37.539007896Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B9716113EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:40:39.074686416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:39.239043999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:40:39.242399873Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B9D6EB398E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:40:39.330863054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:40:39.335700663Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1B9DC7A5F80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:40:40.366152084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:40:40.372322144Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BA1A43FBBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:40:42.92262146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:40:42.926129623Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BAB27CAB76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:40:43.216908574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:40:43.220221631Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BAC403398A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:40:44.055728531Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:40:44.07502945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:40:44.07504293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:44.075059465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:40:45.48489442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:40:45.488958047Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BB4B37F884, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:40:46.168885692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:40:46.173719684Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BB73FBDC07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:40:46.430310857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:40:46.434094697Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BB83939210, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:40:49.075187779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:49.143345129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:40:49.149595954Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BC256D8454, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:40:53.799688576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:40:53.803364286Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BD3AD06872, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:40:54.075091041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:40:58.855653092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:40:58.859097312Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BE6828CB93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:40:59.075205209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:03.060027042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:41:03.064032271Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BF62CB070A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:41:03.645445734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:41:03.653223588Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BF85EA11ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:41:04.075376008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:04.906060599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:41:04.909560798Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1BFD0CBEBCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:41:05.834589252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:41:05.838019308Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C008235CA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:41:09.075281225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:09.373686824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:41:09.377180058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C0DB1602D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:41:13.889161281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:41:13.892703986Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C1E83B818B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:41:14.022037083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:41:14.026262585Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C1F032276D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:41:14.050192417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:41:14.057142174Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C1F207EA9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:41:14.057158702Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:41:14.057181197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:41:14.06011098Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C1F237F100, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:41:14.074811093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:14.074813054Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:41:14.07483402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:41:15.829688417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:41:15.837639065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C25C28E757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:41:19.075131794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:19.839828161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:41:19.846207678Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C34B147102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:41:20.031672728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:41:20.036525299Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C35670A116, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:41:21.501278347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:41:21.507643744Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C3AE1EA351, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:41:21.684189092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:41:21.689485122Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C3B8F72441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:41:22.827670328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:41:22.833551117Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C3FD26ADB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:41:23.643743815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:41:23.648561765Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C42DAA0E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:41:23.64858094Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:41:23.648603803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:41:23.65215073Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C42DEC8B1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:41:24.074419751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:24.179752169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:41:24.183594777Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C44D9EECC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:41:25.556358938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:41:25.562373512Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C49FCE2CF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:41:26.072822848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:41:26.076034236Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C4BE6CF4F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:41:29.075351923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:32.539861986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:41:32.543570403Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C63FE62B36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:41:34.074433779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:34.469994999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:41:34.473821674Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C6B2F3899E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:41:35.743932052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:41:35.747522309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C6FEE432DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:41:36.894909629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:41:36.902818975Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C743B931AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:41:38.725597411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:41:38.72905936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C7B0990941, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:41:38.867728772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:41:38.87091174Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C7B90E2861, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:41:38.937978512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:41:38.942606571Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C7BD4217FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:41:39.075213737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:44.027391959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:41:44.030847415Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C8EC9C60B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:41:44.055686581Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:41:44.074355897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:41:44.074372375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:44.074384562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:41:44.086577905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:41:44.090564644Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C8F023CE1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:41:45.307784134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:41:45.311149463Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C938EBF999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:41:45.598363094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:41:45.601792369Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C94A3ECB2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:41:48.412993188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:41:48.416320637Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C9F201CDC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:41:48.41633495Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:41:48.416353157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:41:48.419038647Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1C9F22E0702, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:41:49.074611915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:41:49.44146022Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB1CA2F1E4F40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:41:52.692548323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:41:52.696412236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CAF11D70EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:41:53.280082075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:41:53.286128338Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CB14443506, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:41:53.63386237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:41:53.637177799Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CB29324653, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:41:54.075351517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:58.590572365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:41:58.594049457Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CC50A52AD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:41:59.016250129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:41:59.024400217Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CC6A4A5560, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:41:59.075337162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:41:59.604676082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:41:59.611032679Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CC8D40B9A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:42:01.371674956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:42:01.376872134Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CCF68255B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:42:01.376894293Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:42:01.37691696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:42:01.383839375Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CCF6EF0E11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:42:02.71825627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:42:02.724344841Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CD46D302E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:42:04.074369025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:07.738508622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:42:07.741935114Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CE71E57899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:42:09.074390724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:12.129791877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:42:12.1332475Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CF77A424F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:42:13.654122841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:42:13.658250549Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1CFD2899C8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:42:14.074400519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:42:14.074405366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:14.074437888Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:42:14.529366124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:42:14.536794805Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D006E330BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:42:15.175974023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:42:15.179692122Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D02D35E1FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:42:16.636271263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:42:16.639665035Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D0843E6EAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:42:16.639681903Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:42:16.639701756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:42:16.642384485Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D0846C025C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:42:17.084620011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:42:17.088000287Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D09EF7790F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:42:17.451352813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:42:17.457292438Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D0B4FBCA47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:42:18.733206935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:42:18.739735955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D10166E454, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:42:19.074544022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:19.939139922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:42:19.94243605Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D1491CBA7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:42:20.385734231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:42:20.390918443Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D163D4CFED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:42:21.798181133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:42:21.804751005Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D1B81AAF8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:42:23.792566505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:42:23.798771023Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D22EF5C1DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:42:24.074401429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:27.346605789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:42:27.34996346Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D302A27429, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:42:29.074846641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:30.044220355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:42:30.047416345Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D3A369ED82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:42:31.937680409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:42:31.94144858Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D4144F4B5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:42:32.229392917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:42:32.235300806Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D425D161A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:42:32.235319845Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:42:32.235338914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:42:32.238177697Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D425FFA4B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:42:33.631875753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:42:33.636306121Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D479433AFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:42:34.074916772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:34.573280838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:42:34.576683515Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D4B1602D8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:42:36.079465508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:42:36.083069438Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D50B2AD77B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:42:36.477202877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:42:36.48536805Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D5231BB7C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:42:36.485390021Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:42:36.485413267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:42:36.489047694Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D52355FEAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:42:38.684424781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:42:38.687993714Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D5A66C5814, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:42:39.075230711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:39.95526334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:42:39.958750541Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D5F22A4EAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:42:41.856041738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:42:41.859487057Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D663775F1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:42:43.007750868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:42:43.01124533Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D6A81D689E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:42:43.145670547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:42:43.149037411Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D6B053DD02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:42:44.05624306Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:42:44.07472477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:44.07475178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:42:44.074780354Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:42:44.318676187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:42:44.322119186Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D6F63F59EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:42:44.407342295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:42:44.410786169Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D6FB87B55C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:42:47.848489401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:42:47.851845845Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D7C8A36E03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:42:49.074533487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:52.156737529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:42:52.828929625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:42:52.83581635Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D8F1AFBC79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:42:53.470838871Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D9178DEA15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:42:54.074522199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:55.652673104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:42:55.660527262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D99A1042E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:42:57.105688807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:42:57.113803964Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D9F0AF912E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:42:57.113826293Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:42:57.113849738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:42:57.116811076Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1D9F0E12C1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:42:57.908195853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:42:57.912358151Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DA2049A850, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:42:59.074880871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:42:59.517076569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:42:59.523156031Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DA804E74DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:43:00.446967375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:43:00.45029039Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DAB790B3C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:43:04.074863585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:07.126127174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:43:07.129966819Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DC45B52FF4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:43:07.595247576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:43:07.601203554Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DC61CB1C52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:43:08.101674214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:43:08.105053274Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DC7FD10A0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:43:08.632252842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:43:08.637489766Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DC9F903B70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:43:09.074689911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:11.392526694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:43:11.398419809Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DD441F6B4B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:43:11.847433897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:43:11.850688316Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DD5F1571B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:43:13.177050262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:43:13.181087096Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DDAE5E26D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:43:13.639296787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:43:13.642850464Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DDC9E485D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:43:14.074978797Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:43:14.07501085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:43:14.075006897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:14.39708101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:43:14.400732089Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DDF7106A78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:43:19.075016973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:19.791979163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:43:19.795733156Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DF38A52901, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:43:20.523193174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:43:20.528834061Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DF64555733, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:43:20.52885425Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:43:20.528873374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:43:20.531894368Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DF648515C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:43:22.452694084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:43:22.456142835Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1DFD736CC1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:43:23.410615796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:43:23.414018955Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E0104E1987, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:43:23.414037615Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:43:23.414060067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:43:23.416882715Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E0107C056B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:43:24.074489943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:26.509164131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:43:26.512409924Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E0C8FD0757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:43:26.581595918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:43:26.584881283Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E0CD4D98A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:43:26.584898484Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:43:26.584918263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:43:26.587647765Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E0CD79F419, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:43:27.561007683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:43:27.564341005Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E107AF476C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:43:27.938339965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:43:27.941564532Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E11E2C6713, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:43:28.227246834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:43:28.231094317Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E12F6DD925, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:43:28.231109934Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:43:28.231129782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:43:28.237230562Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E12FCD098B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:43:29.074526736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:31.52006747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:43:31.526088909Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E1F3D15327, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:43:32.550118979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:43:32.554619779Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E2311280E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:43:33.837056628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:43:33.844909716Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E27E0A530E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:43:34.074751029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:37.786753225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:43:37.793388448Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E369616230, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:43:38.780964411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:43:38.784437523Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E3A474BE75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:43:38.784455863Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:43:38.784476369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:43:38.78729192Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E3A4A2FFDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:43:39.075108188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:39.719499135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:43:39.722898084Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E3DC644179, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:43:41.271818618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:43:41.275207118Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E438EB2A79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:43:41.493801156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:43:41.497470663Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E44626D7A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:43:42.257942616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:43:42.26135802Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E473B1C2AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:43:43.380899485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:43:43.388297126Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E4B6DD084C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:43:44.055822568Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:43:44.075196932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:44.075221294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:43:44.075255969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:43:46.131622964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:43:46.136682341Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E55AADF3D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:43:48.011278896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:43:48.018909104Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E5CAE035B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-06-29T23:43:49.074727386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:51.731908392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:43:51.739772985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E6A8A7BEF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:43:52.25726073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:43:52.264258143Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E6C7EB56C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:43:52.851123677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:43:52.856788492Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E6EB3BF123, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:43:54.074821127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:58.342855627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:43:58.346156188Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E8326D3B75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:43:58.660027383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:43:58.66327036Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E845539CEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:43:59.074748549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:43:59.512855105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:43:59.516579968Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1E8782FF616, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:44:04.074785681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:06.917736151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:44:06.925766174Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EA31CC4102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:44:07.438685618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:44:07.442612053Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EA5098789A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:44:07.442641781Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:07.442662672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:44:07.446080406Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EA50CDE8E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:44:09.074570367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:10.130991533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:44:10.134388793Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EAF10FC658, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:44:10.134404599Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:10.13442203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:44:10.137243223Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EAF13D599A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:44:10.669298288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:44:10.673218843Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EB112CF3AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:44:10.931832568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:44:10.937668933Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EB20F0524A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-06-29T23:44:12.112350338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:44:12.116325644Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EB672CFF8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:44:13.434308789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:44:13.437898489Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EBB5F6E7F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:44:13.700803243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:44:13.704877238Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EBC5D73BCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:44:14.075268547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:14.075273525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:44:14.075302451Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:44:14.081849252Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-06-29T23:44:14.084035366Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008898648s
level=info ts=2026-06-29T23:44:14.260094013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:44:14.263488791Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EBE72AE70D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:44:16.525993914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:44:16.529337018Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EC6E3B5E94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:44:17.574375138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:44:17.577846393Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ECACBA0D5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:44:17.577865433Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:17.577886297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:44:17.580729431Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ECACE71C4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:44:18.346559084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:44:18.354949898Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ECDB0C9DA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:44:18.716924337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.6 kB" avg_comp="3.6 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:44:18.723165925Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ECF0FDA2B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:44:19.075382553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:20.066827471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:44:20.070702071Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ED414B726C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:44:21.531279465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:44:21.535080627Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1ED989373AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:44:22.136085204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:44:22.139648284Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EDBCA09C5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:44:22.139667568Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:22.139689976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:44:22.142585008Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EDBCCF91C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:44:24.074451741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:24.116281778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:44:24.120238782Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EE32ABB732, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:44:24.213310879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:44:24.216804223Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EE386E988F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:44:24.69414142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="793 B" avg_comp="793 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:44:24.697502409Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EE55177C68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:44:26.665896301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:44:26.66999508Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EECAA92470, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:44:28.235994642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:44:28.241789879Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EF28573BDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:44:28.668606561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:44:28.673308078Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EF41FFB4FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:44:28.988322439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:44:28.99160152Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EF55090064, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:44:28.99162186Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:28.991658465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:44:28.994405946Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EF5537044D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:44:29.075047963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:31.753668828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:44:31.757056617Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EFF9DEAE9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:44:31.757072666Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:44:31.757091405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:44:31.759816176Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1EFFA0B6311, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:44:32.949110522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:44:32.953208301Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F0412C2EA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:44:34.074474394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:37.248507901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:44:37.254939368Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F1418F2466, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:44:39.075323189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:44.056396805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:44:44.074195906Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:44:44.074466009Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-06-29T23:44:44.074477869Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:44:44.074498998Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.848µs success=true
level=info ts=2026-06-29T23:44:44.074528076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:44:44.074512416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:44.074548647Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:44:45.62991201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:44:45.633350586Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F334F70876, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:44:45.839140324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:44:45.84246064Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F3416EA216, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:44:46.043055901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:44:46.048011982Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F34DAF60E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:44:46.689896331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:44:46.691021772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:44:46.695731987Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F374497611, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:44:46.69573414Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F3744A8DD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:44:47.023487588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:44:47.027119526Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F38806B70A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:44:47.463777475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:44:47.469174126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F3A26430F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:44:49.074878593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:51.10767858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:44:51.111446198Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F47B7CF798, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:44:52.317156056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:44:52.323876728Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F4C3BA1AF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:44:54.074538672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:44:54.410966934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:44:54.414491872Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F5405C2BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:44:55.328613094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:44:55.332459876Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F5771381A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:44:55.476994229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:44:55.483768419Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F58016ED80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:44:57.783478346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:44:57.787055868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F60960E55A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:44:58.654446152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:44:58.658170342Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F63D4A3C49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:44:59.0748583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:00.65290047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:45:00.65660222Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F6B468582B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:45:04.075335725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:05.979008815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-06-29T23:45:05.982498949Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F7F1DE073E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-06-29T23:45:05.982516654Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:45:05.982537916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="124 B" avg_comp="124 B" total_uncomp="49 kB" avg_uncomp="49 kB" max_age=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:45:05.985704436Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F7F20F3E81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:45:06.277559541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:45:06.281197975Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F803A7E331, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:45:07.08255305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:45:07.085896975Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F833A2698B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:45:07.527234736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:45:07.530541093Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F84E23809D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:45:08.609674025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:45:08.612995723Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F88EA7408D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:45:09.074735564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:11.312412057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:45:11.315890083Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F92FC34681, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:45:11.619605162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:45:11.62312403Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F942123847, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:45:12.687355708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:45:12.690673645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F981B529CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:45:12.850030893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:45:13.246521161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1F9A2D64D36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:45:14.07459156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:14.074596085Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:45:14.074648433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:45:14.883268632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:45:14.889263822Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FA04BD48E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:45:18.620674226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="909 B" avg_comp="454 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" idle=1 full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:45:18.624059089Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FAE35BB5EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:45:19.075266604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:21.067737016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:45:21.072450576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FB753AA0A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:45:24.07437844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:24.807145083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:45:24.810428484Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FC54181EA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:45:25.185257044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:45:25.189158098Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FC6AAB033A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:45:26.760945664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:45:26.768539454Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FCC8CFF8C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:45:26.768561366Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:45:26.768584228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:45:26.774464774Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FCC92A6DFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:45:28.77912826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:45:28.785452357Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FD41047C7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:45:29.074980637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:29.79759216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:45:29.800856933Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FD7D8B9745, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:45:34.074415541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:34.862750122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:45:34.868353833Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FEAB98401D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:45:34.868378318Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:45:34.868401918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:45:34.876327022Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FEAC1298BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:45:38.009494695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:45:38.01540151Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FF672E1860, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:45:38.015421021Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:45:38.015443432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:45:38.021109565Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FF67841C78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:45:38.467562301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:45:38.475282676Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB1FF8294EA6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:45:39.075065603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:41.535208214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:45:41.541065036Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20039531C88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:45:43.346074439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:45:43.350240773Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB200A5291939, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:45:44.055577267Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:45:44.075216268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:44.075227593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:45:44.075257613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:45:45.126222128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:45:45.132232023Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2010F5F000F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:45:45.761185303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:45:45.765212392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2013514735D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:45:46.362740385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:45:46.365975726Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20158E84C17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:45:47.414394174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:45:47.417773505Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20197991218, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:45:49.074961991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:51.188485529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:45:51.192223691Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20278908545, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:45:51.43774395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:45:51.441469436Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB202876EFA06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:45:51.636037245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:45:51.644143851Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2029383A1D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:45:51.923086033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:45:51.928938733Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB202A47C9298, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:45:53.191590978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:45:53.195183654Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB202EFF64555, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:45:53.54104391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:45:53.547446483Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20304EFD382, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:45:54.075131719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:45:56.773686081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:45:56.777053633Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB203C573AA0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:45:57.080044687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:45:57.083378745Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB203D7B6C384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:45:57.099455789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:45:57.102860036Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB203D8DF69B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:45:59.074855693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:03.364986688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:46:03.368323893Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2054E530F68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:46:04.034857402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:46:04.038210673Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2057641D4EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:46:04.074607141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:46:04.717795833Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2059EC54B5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:46:06.807244002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:46:06.810698951Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2061B812173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:46:09.074402306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:10.858139698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:46:10.861552387Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2070CF50786, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:46:11.771555779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:46:11.781756174Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20743CDD12E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:46:11.781777909Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:46:11.781802632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:46:11.793013632Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB207447BE394, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:46:12.52103897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:46:12.527206088Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB207703C420E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:46:13.865051378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:46:13.871280684Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB207C05A5104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:46:14.074680009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:14.074710324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:46:14.074741613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:46:15.089519361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:46:15.092824893Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20809296FCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:46:17.432086805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:46:17.436920793Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20894CE7D02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:46:18.382526169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:46:18.386768066Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB208CD7E39C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:46:19.074337952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:22.098418043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:46:22.110654944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB209AB746668, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:46:24.074359034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:24.352669814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:46:24.359098074Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20A317619B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:46:25.266741085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:46:25.271261247Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20A67DA44B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:46:26.703033186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:46:26.711793761Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20ABDB44573, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:46:27.532261029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:46:27.538126321Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20AEEF68E71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:46:28.754261693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:46:28.757716549Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20B37A67426, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:46:28.757733938Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:46:28.757751645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:46:28.760544457Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20B37D4F9F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:46:29.074817044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:29.404140986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:46:29.406470263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:46:29.407700735Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20B5E628355, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:46:29.409588877Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20B5E84975F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:46:34.074864969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:34.829495297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:46:34.835788156Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20CA1EDD5BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:46:36.069458907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:46:36.073304828Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20CEBAC602B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:46:38.403656565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:46:38.407645882Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20D76CE774A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:46:39.075290324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:41.416260237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:46:41.419789909Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20E2A5C8E7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:46:42.455617615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:46:42.459045174Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20E6850D036, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:46:42.979251181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:46:42.983325156Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20E8789D4FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:46:43.931199256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:46:43.934607607Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20EC043DFEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:46:44.055728395Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:46:44.074421965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:46:44.074453042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:46:44.074485974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:44.284408749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:46:44.288235016Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20ED5585B6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:46:44.288256174Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:46:44.288278738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:46:44.29451203Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20ED5B89C4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:46:45.365305626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:46:45.371391395Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20F15E76F24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:46:46.401428793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:46:46.409273359Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB20F53C54948, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:46:49.075299037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:51.468826368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:46:51.474836829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21081B26926, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:46:52.802346484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:46:52.805799222Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB210D1072DEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:46:53.138671484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:46:53.142146813Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB210E5129850, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:46:54.074760033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:55.647762951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:46:55.651112786Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2117A9F523C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:46:57.316398107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:46:57.320324347Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB211DE1CC203, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:46:59.074960474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:46:59.553897341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:46:59.559852223Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2126398357D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:47:02.867207289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:47:02.870819244Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21328F34E37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:47:03.735056302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:47:03.738257123Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2135CA7D0B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:47:04.075224177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:04.819595625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:47:04.822943587Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2139D4E1A25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:47:06.93707554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:47:06.941958439Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2141B884704, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:47:06.941978065Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:47:06.941998293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:47:06.945600296Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2141BCD9FE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:47:09.074363912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:09.220679694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:47:09.224808061Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB214A3ACF72F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:47:10.716650112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:47:10.724008906Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB214FD0A60DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:47:12.460014579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:47:12.463426843Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21564B86D6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:47:14.074381497Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:47:14.074412367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:47:14.074425359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:16.342736663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:47:16.346039356Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2164C23DECF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:47:16.538660841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:47:16.542721263Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21657DD7338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:47:18.650874525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:47:18.657092665Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB216D5DEE97C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:47:19.075011886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:20.379828714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:47:20.385081727Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2173CE303B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:47:20.73907568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:47:20.746427612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB217526B8FDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:47:21.113673001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:47:21.121024155Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21768AC7046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:47:22.081673324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:47:22.084928143Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB217A233EE03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:47:23.487874309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:47:23.491173481Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB217F605B318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:47:24.074669612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:27.366531362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:47:27.370332034Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB218DD3BD63F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:47:28.615544959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:47:28.621907606Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21927D56957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:47:29.074698702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:29.154007949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:47:29.157957284Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21947C639E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:47:29.867434001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:47:29.871193738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB219724D75B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:47:33.594816984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:47:33.60638666Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21A50EFDA8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:47:34.074663222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:34.39283807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:47:34.398473248Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21A8026197D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:47:34.398493192Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:47:34.398513929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:47:34.401192614Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21A8052048A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:47:34.918464506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:47:34.922106073Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21A9F5B1057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:47:34.990411735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:47:34.994191862Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21AA3A33FED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:47:39.074393444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:40.35261675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:47:40.355933356Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21BE33DFBDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:47:40.671604513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:47:40.67487562Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21BF6405959, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:47:42.096777553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:47:42.100170098Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21C4B347F59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:47:44.055605117Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:47:44.07519768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:47:44.075222613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:47:44.075219167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:47.687041221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:47:47.69026328Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21D9866B515, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:47:48.251467131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:47:48.254608715Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21DBA0A5A47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:47:48.611678214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:47:48.616103472Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21DCF86D521, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:47:48.707441048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:47:48.710729727Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21DD5393F5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:47:49.074611751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:53.142087821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:47:53.145834133Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21EDD90E1A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:47:54.075092152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:47:56.864160408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:47:56.867391466Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB21FBB676223, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:47:59.074597265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:00.640968849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:48:00.644191604Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2209C8423B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:48:01.140746479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:48:01.14397056Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB220BA4E8914, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:48:02.074849949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:48:02.079882448Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB220F216F086, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:48:02.258293271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:48:02.2659684Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB220FD2E1127, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:48:02.732078178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:48:02.737949848Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB221194FC964, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:48:03.722653481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:48:03.726055331Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2215434FFF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:48:04.075222693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:04.978326595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:48:04.982431566Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2219F11F1EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:48:05.529669856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:48:05.533229499Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB221BFEB084D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:48:07.357251285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:48:07.362791871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2222CF90E03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:48:08.804617934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:48:08.811800026Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22283556021, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:48:09.075347547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:11.440289814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:48:11.446141021Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB223205DE9CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:48:12.893726853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:48:12.90002168Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2237705B849, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:48:14.074884374Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:48:14.074916928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:48:14.074920614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:15.82004691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:48:15.82621376Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB224256D29BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:48:16.012680169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:48:16.016469723Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22430C1A13A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:48:16.016493949Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:48:16.016521869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:48:16.020114687Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22430F83513, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:48:19.075056791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:20.898032236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:48:20.901431853Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22553F1271C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:48:23.397585474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:48:23.406943536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB225E9479198, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:48:24.074713977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:24.102814214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:48:24.109596223Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22613242E7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:48:24.756243242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:48:24.759647327Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22639E782C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:48:27.066833226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:48:27.070264389Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB226C3A12548, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:48:29.07470776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:30.884826631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:48:30.888793837Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB227A735D5D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:48:31.340606531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:48:31.34388867Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB227C25C2F20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:48:31.487412905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:48:31.490669814Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB227CB1B131D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:48:34.074616695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:38.664247304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:48:38.667646025Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22976E30501, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:48:39.075379818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:39.606672784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:48:39.610032528Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB229AF0F2C9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:48:44.055852351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:48:44.074507674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:48:44.074536082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:48:44.074539178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:44.881674549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:48:44.886779629Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22AE97ED87F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:48:45.191167249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:48:45.194525193Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22AFBEAFD93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:48:46.188689628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:48:46.192199469Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B37633AC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:48:46.565263254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:48:46.570214629Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B4DEB72EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:48:46.570237889Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:48:46.570267634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:48:46.578300139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B4E6824B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:48:47.114836402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:48:47.120920852Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B6EBE02CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:48:47.494493329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:48:47.498368533Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B85390F77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:48:47.498387636Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:48:47.498408731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:48:47.501849017Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22B856DA940, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:48:49.075215926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:49.793229489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:48:49.796895141Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22C0E3CA856, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:48:53.591747637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:48:53.595227579Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22CF0A41E1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:48:54.074354801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:48:56.152259182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:48:56.155614834Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22D893FCDA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:48:56.155650432Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:48:56.155670642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:48:56.158421888Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22D896E5585, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:48:58.589883737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:48:58.593366668Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22E1A8DC513, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:48:58.593386729Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:48:58.59340836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:48:58.59622772Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22E1ABC7C5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:48:59.075174435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:00.257269634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:49:00.261557984Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22E7DF5033B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:49:00.270564679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:49:00.27804962Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22E7EF88FC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:49:00.634912652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:49:00.63882153Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22E9478EAFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:49:03.350589411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:49:03.358978289Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22F369A785B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:49:04.074738883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:04.373757071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:49:04.380316972Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22F737BB340, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:49:04.916301611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:49:04.922787323Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22F93CCFBFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:49:06.202677764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:49:06.207719936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22FE067481E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:49:06.568732557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:49:06.575223873Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB22FF64CBB71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:49:07.27427005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:49:07.278093014Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23020332053, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:49:07.526775511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:49:07.533705019Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2302F6ED8F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:49:08.795673551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:49:08.799086638Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2307ADBD725, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:49:09.074669411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:09.381497551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:49:09.385041307Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2309DC6EDF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:49:10.354551896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:49:10.358460646Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB230D7C9F772, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:49:10.707293952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:49:10.710764546Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB230ECCEB63B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:49:12.138675633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:49:12.142486718Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB231421FC44D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:49:12.142506847Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:49:12.142529828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:49:12.145429405Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23142540FF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:49:12.770098694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:49:12.778141876Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23168084857, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:49:14.075237059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:49:14.075267933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:49:14.0752786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:14.07788664Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-06-29T23:49:14.080083331Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005464521s
level=info ts=2026-06-29T23:49:16.849672664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:49:16.855988677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2325B168444, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:49:16.856007897Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:49:16.85602851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:49:16.858965517Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2325B4638ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:49:17.749965863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:49:17.753309074Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2329093DCD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:49:19.074831831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:23.772979721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:49:23.776824418Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB233F799432E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:49:24.074528255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:25.341876321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:49:25.345251472Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23455166818, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:49:26.94578583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:49:26.94964431Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB234B4B1A032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:49:29.074702255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:30.009775335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:49:30.013839434Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2356B54BD31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:49:31.369212216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:49:31.374061706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB235BC5B6B73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:49:32.579450371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:49:32.582926106Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB236047D4ED0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:49:33.580263157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:49:33.584029335Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2364026D03C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:49:34.075226748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:49:39.074994696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:39.237350251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:49:39.24078484Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2379153227A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:49:40.689743856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:49:40.693184316Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB237E7E64CA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:49:42.679186635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:49:42.685028854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2385E9B0030, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:49:42.971149147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:49:42.978735656Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2387020DB62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-06-29T23:49:43.043744689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:49:43.04944301Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2387457BE88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:49:44.056281682Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:49:44.074310422Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:49:44.074344718Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:49:44.074375684Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:49:44.074388828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:44.074436501Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-06-29T23:49:44.074444222Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:49:44.074462242Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.001µs success=true
level=info ts=2026-06-29T23:49:44.532048872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:49:44.535319502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB238CCE876FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:49:44.857970942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:49:44.861322665Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB238E055D731, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:49:46.99638255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:49:47.000406237Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2395FD5E95F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:49:47.660390282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:49:47.666638486Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB239878BE22D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-29T23:49:49.074617841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:52.410016807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:49:52.415814443Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23AA29EC2D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:49:53.486567604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:49:53.489901696Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23AE2A45214, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:49:54.075371068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:49:54.223676116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:49:54.229183841Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23B0EB56F25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:49:55.847472651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:49:55.853682425Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23B6F8845CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:49:57.577687591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:49:57.581354291Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23BD680FDBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:49:57.955463018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:49:57.958847727Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23BED0415C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:49:59.074675203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:50:00.816708526Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB23C975C1659, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:50:01.399466343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:50:01.402860484Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23CBA4A8964, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:50:01.965516817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:50:01.969382709Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23CDC0B9E60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:50:04.074429051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:04.884825059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:50:04.888171619Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23D8A095190, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:50:05.33403932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:50:05.338057651Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23DA4D9C72F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:50:05.978018788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:50:05.984950313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23DCB6219E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:50:07.165775447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:50:07.169273102Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23E11FDE83B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:50:07.169289307Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:07.169308855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:50:07.172094864Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23E122C6640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:50:07.343546629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:50:07.346937433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23E1C957B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:50:08.298650291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:50:08.306727634Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23E55C9D01C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:50:09.074674203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:12.451901578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:50:12.458131866Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23F4D39E366, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:50:14.075301086Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:50:14.075333906Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:50:14.075350912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:15.236756858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:50:15.240107022Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB23FF30E903F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:50:19.07437466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:19.234672972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:50:19.237994133Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB240E15A43EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:50:21.338678815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:50:21.343013904Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2415EC4EB0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:50:23.740224499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:50:23.743655497Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB241EDE83D0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:50:23.743670218Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:23.743687712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:50:23.746366873Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB241EE14F826, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:50:24.075231962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:29.053789962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:50:29.058715069Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2432AB61111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:50:29.075314849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:30.251839586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:50:30.259340506Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24372450E36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:50:30.527922367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:50:30.533604219Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB243829F6EB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:50:30.533639694Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:30.5336609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:50:30.536191832Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24382CA1A5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:50:33.225748951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="380 kB" avg_comp="380 kB" total_uncomp="2.3 MB" avg_uncomp="2.3 MB" full=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:50:33.229377787Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB244234DE9C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-06-29T23:50:33.229395336Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:33.229415896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="113 kB" avg_uncomp="113 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:50:33.235030818Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24423A4440C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:50:34.074958504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:34.80339286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:50:34.807677522Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24481600F9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:50:35.29997761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:50:35.305820205Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2449F10ECFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:50:35.305840931Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:35.305863385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:50:35.308701231Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2449F3E59C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:50:35.506331605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:50:35.509564296Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB244AB3623F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:50:35.927658277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:50:35.930806814Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB244C4512A96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:50:38.18369391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:50:38.187061401Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2454ACCAAE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:50:38.187078907Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:38.187099892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:50:38.189836468Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2454AF9B4C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:50:38.318229671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:50:38.3215742Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24552D1AA4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:50:39.074685709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:39.855603321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:50:39.858934784Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB245AE73CF38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:50:39.914009473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:50:39.917212302Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB245B1EE2A2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:50:43.136096135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:50:43.139860676Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24671FE9080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:50:43.856028272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="909 B" avg_comp="454 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:50:43.859354897Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2469CE528EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:50:43.984681018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:50:43.987797948Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB246A48D1753, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:50:43.987816439Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:43.987843949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:50:43.990583228Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB246A4BA01A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:50:44.055615641Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:50:44.074964027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:50:44.07499776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:44.07500672Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:50:45.062158485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:50:45.065260685Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB246E4C62157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:50:46.226383024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:50:46.230085492Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2472A2FAF92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:50:47.676965822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:50:47.680201638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24780A3690C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:50:47.934219479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:50:47.938110105Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2478FFD6A42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:50:47.938129281Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:50:47.938149048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:50:47.941438789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24790320E15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:50:48.542321721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:50:48.545495681Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB247B436B380, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:50:49.075016805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:49.580728994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:50:49.584084953Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB247F21D57F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:50:50.188129437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:50:50.193746356Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2481674091B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:50:50.40774427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:50:50.414458053Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB248239E3867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:50:52.069785759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:50:52.075715072Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB248869F7598, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:50:52.470820508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:50:52.478874815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2489EA7D504, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:50:54.075100771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:50:54.9939979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:50:54.999957354Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24934ED3659, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:50:56.372743338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:50:56.377478867Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24986F27C16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:50:58.269296802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:50:58.272681443Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB249F7FF8599, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:50:59.075098681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:01.225360148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:51:01.229230906Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24AA8346417, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:51:04.075013623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:06.013201564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:51:06.016602552Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24BC5914F35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:51:06.096903488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:51:06.100213545Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24BCA8CF45B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:51:06.100234238Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:51:06.10025843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:51:06.103000715Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24BCABA3FD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:51:07.529855038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:51:07.533080736Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24C1FF61155, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:51:09.001167782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:51:09.004912952Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24C77AD0A4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:51:09.075391523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:10.557673627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:51:10.563316846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24CD49156B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:51:14.074306532Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:51:14.074344552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:51:14.074392778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:14.376283654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:51:14.383620382Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24DB847614E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:51:14.383657918Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:51:14.383682994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:51:14.386649904Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24DB877BB95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:51:17.466967532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:51:17.470355342Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24E704408DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:51:18.549104229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:51:18.552505185Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24EB0C355D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:51:19.074694527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:19.442905328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:51:19.447673152Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24EE60AE057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:51:20.257255384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:51:20.260541598Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24F169330CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:51:20.635210856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:51:20.639008574Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24F2D210952, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:51:21.760430717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:51:21.766820438Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24F705AB5BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:51:21.766839408Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:51:21.766861292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:51:21.769681085Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24F70894310, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:51:23.015743504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:51:23.020154175Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24FBB0E3692, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:51:23.475645118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:51:23.48182409Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB24FD690C757, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:51:24.074363821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:24.711839974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:51:24.715059023Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB250201602DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:51:25.539848566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:51:25.543181473Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2505171F133, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:51:29.074696032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:29.564660652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:51:29.567893213Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2514156B1C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:51:30.476342516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:51:30.479575364Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25177ADA64E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:51:31.468300358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:51:31.471440983Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB251B2CD92AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:51:32.460240246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:51:32.463463389Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB251EDEDA010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-06-29T23:51:34.074358906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:34.783369101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:51:34.786734551Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2527866A0A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:51:34.786752697Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:51:34.786773371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:51:34.789441447Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25278934CB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:51:35.113861738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:51:35.117384991Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2528C1AA9B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:51:37.529523669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:51:37.532809459Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2531C15B095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-06-29T23:51:39.074556056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:41.341227164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:51:41.347186499Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB253FF6FF749, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:51:41.465692489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:51:41.473271605Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25406F15267, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:51:44.05559152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:51:44.075003724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:51:44.075037156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:51:44.075041882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:45.212380636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:51:45.216655382Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB254E60B76C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:51:45.417763012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:51:45.421885793Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB254F247B4A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:51:47.78852941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:51:47.795141259Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2557FC36F53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:51:48.712638474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:51:48.715878238Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB255B6A5BD91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:51:49.075346864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:50.21684859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:51:50.220213747Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB256104FBDE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:51:51.289033027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:51:51.292316266Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25650371F2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:51:51.585036382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:51:52.574776408Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2569CA5D899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:51:54.074850222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:54.337742532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:51:54.344246563Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB257061EBFCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:51:58.453480227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:51:58.457158724Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB257FB44ADC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-06-29T23:51:59.074669569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:51:59.113844506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:51:59.117646722Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB258229EF73C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:52:04.074401643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:05.033425909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:52:05.038386842Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB259838C8403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:52:05.141040537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:52:05.148027673Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2598A14B62C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:52:09.07466003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:09.791364804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:52:09.79496366Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25A9F0E7A7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:52:12.852103436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:52:12.855987456Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25B557DB71B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:52:14.010388962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:52:14.014529533Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25B9A90D347, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:52:14.014547703Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:52:14.014571035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:52:14.021500203Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25B9AF4B461, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:52:14.074990006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:14.075022504Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:52:14.075064342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:52:14.478861355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:52:14.48436308Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25BB6924351, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:52:14.594591869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:52:14.599360934Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25BBD59F182, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:52:14.911056538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:52:14.912612812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:52:14.914362222Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25BD033BEE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:52:14.915858048Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25BD04B4562, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:52:15.231059716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:52:15.236597334Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25BE36627D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:52:15.738856588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:52:15.742116593Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25C01898EDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:52:17.522908493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:52:17.5261597Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25C6BDEB97C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:52:19.074370256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:23.727672013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:52:23.731097701Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25DDDB718F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:52:24.075067124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:24.216573144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:52:24.220131944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25DFADBD8A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:52:24.963678807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:52:24.969164635Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25E2781C4F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:52:24.969191228Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:52:24.969222549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:52:24.977407318Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25E28022464, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:52:28.765519063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:52:28.772392924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB25F0A3248B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:52:29.07480548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:33.218913823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:52:33.225287651Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB260139982B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:52:33.534230782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:52:33.538112895Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB260263D093E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-06-29T23:52:34.075123776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:34.748192051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:52:34.751574318Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2606E95D110, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:52:37.008015851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:52:37.01127351Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB260F545DCC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:52:37.147675339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:52:37.153123822Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB260FDBA691F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:52:39.074525802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:40.7766088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:52:40.782312962Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB261D60BA95F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:52:42.180199472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:52:42.183756892Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2622992F043, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:52:42.207802412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:52:42.211387365Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2622B3A1E88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:52:42.211402109Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:52:42.211421793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:52:42.217260047Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2622B944F61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:52:44.055984573Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-29T23:52:44.074906631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:44.074926202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:52:44.074963782Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:52:44.416547865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:52:44.42314052Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB262AF09E5B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:52:45.974758324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:52:45.978179264Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2630BBD9609, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:52:46.982777588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:52:46.986144324Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26347D26561, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:52:48.63094342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:52:49.07506876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-06-29T23:52:50.098219392Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB264015217A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:52:50.098247172Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:52:50.098283745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:52:51.821124454Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26468028A95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:52:53.639276805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:52:53.645477329Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB264D4C05C94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-06-29T23:52:54.074365601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:54.718386185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:52:54.722423791Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26514EFEC76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:52:57.372453453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:52:57.37978779Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB265B34325C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:52:59.075026832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:52:59.115126882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:52:59.120694217Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2661B193339, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:52:59.39788548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:52:59.404345706Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2662C026F86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:53:00.039293498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:53:00.044923352Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2665230440F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:53:00.300106714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:53:00.303610855Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB266619BBF59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:53:01.175034495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:53:01.178839238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26695C700DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:53:01.178860542Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:53:01.178884814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:53:01.184863637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26696233D8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:53:01.279337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:53:01.284740493Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2669C16DC2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:53:02.115916603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:53:02.121213589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB266CDF2FBFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:53:02.297755264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:53:02.30347421Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB266D8CF1F46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:53:02.774665061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:53:02.7780553Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB266F51781EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:53:04.075224433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:04.65228308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:53:04.655945498Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2676503B7D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:53:05.647943864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:53:05.651344921Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB267A05B38A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:53:07.548598613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:53:07.556816462Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26811EBD955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:53:08.325011766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:53:08.331137328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2684017008F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:53:09.075269459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:11.417423743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:53:11.42080415Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB268F83EF3D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:53:14.074561257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:53:14.074589297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:53:14.074603591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:16.555726925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:53:16.559372132Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26A2A87A1FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:53:16.689773236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:53:16.69604647Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26A32A82EF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:53:17.246748608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:53:17.250578299Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26A53B9EF32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:53:19.074999157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:20.748891456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:53:20.754749489Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26B24965C6A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:53:22.865653323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:53:22.869038658Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26BA29CCFD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:53:24.075083478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:53:29.075138996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:29.61950673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:53:29.623491411Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26D352EC6F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:53:30.902059531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:53:30.90569554Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26D819E9C89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:53:30.905712356Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:53:30.905732953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:53:30.908671181Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26D81CFF640, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:53:31.756174917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:53:31.765496778Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26DB4E148AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:53:34.075414866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:34.383503071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:53:34.389604613Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26E514A0389, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:53:34.709933284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:53:34.713355563Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26E6496B5CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:53:35.035527711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:53:35.039473758Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26E78054338, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:53:37.340346546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:53:37.344548851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26F01685D89, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:53:37.731668901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:53:37.737681484Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26F18D9D8A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:53:38.805553734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:53:38.809080107Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26F58B63844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:53:39.074579395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:40.731639701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:53:40.735395286Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB26FCB841C85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:53:42.196413752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:53:42.199909531Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27022D0FD71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:53:42.258100038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:53:42.261323005Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB270267BF69E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:53:43.012261203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:53:43.017755599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27053912F96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:53:44.05574741Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:53:44.074968969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:53:44.075000858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:53:44.074997591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:44.337223434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:53:44.344280437Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB270A29311DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:53:44.344304012Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:53:44.344328076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:53:44.347764292Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB270A2D3ACAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:53:46.235612763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:53:46.240490928Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27113A90B8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:53:46.694568276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:53:46.700783744Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2712F1827D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:53:46.961531047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:53:46.964982442Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2713ED6ABD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:53:49.075349348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:51.203679955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:53:51.210869395Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2723BE8D43F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:53:52.769385405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:53:52.776008809Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB272993368F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:53:54.075301578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:53:54.414437403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:53:54.418061278Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB272FB1483F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:53:55.971083145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:53:55.97436368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27357D7AC2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:53:56.917190371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:53:56.92193886Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27390415D17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:53:57.93284794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:53:57.936296488Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB273CCC84E02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:53:59.07533102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:03.550156399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:54:03.554197092Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2751B9DDD4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:54:04.074881333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:04.764019834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:54:04.767841435Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27563F8C6D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:54:07.949521388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:54:07.956890204Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB276220C0238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:54:09.075049768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:09.447927554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:54:09.451284611Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2767B218901, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:54:10.342812578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:54:10.346478182Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB276B078997D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:54:13.999057448Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2778A3450E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:54:14.074599687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:54:14.074650773Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:54:14.074662691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:14.080373391Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-06-29T23:54:14.082543743Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007965148s
level=info ts=2026-06-29T23:54:15.783191074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:54:15.791023696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB277F50219CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:54:17.554658931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:54:17.560451962Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2785E780844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:54:17.97775941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:54:17.980953293Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB278778A64A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:54:17.980972157Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:54:17.980991484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:54:17.983761556Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27877B6614E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:54:18.127266696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:54:18.131103184Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2788077F55A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:54:18.66273326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:54:18.665907328Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB278A05DBE68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-06-29T23:54:19.074713721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:19.45227948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:54:19.456826273Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB278CF70F6DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:54:20.422227696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:54:20.425551652Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB279093EE294, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:54:24.075091395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:24.511984556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:54:24.515716161Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB279FD0591DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:54:24.837434819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:54:24.840926604Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27A106C2169, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:54:29.075300392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:29.565395131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:54:29.568763438Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27B2A395E9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:54:29.946502998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:54:29.949930109Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27B40F13B3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:54:31.280459853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:54:31.283757687Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27B9071DBD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:54:31.893599601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:54:31.896861029Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27BB4FDFCC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:54:32.930850476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:54:32.934469585Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27BF2D6B56B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:54:33.611397361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:54:33.620685524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27C1BBD4A1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-29T23:54:34.074754906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:37.290458188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:54:37.297332606Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27CF6E21C66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:54:38.42626159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" max_age=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:54:38.434478849Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27D3AA8C59F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:54:39.075006396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:39.181377787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:54:39.187452426Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27D678BDCA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:54:43.916912521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:54:43.920294674Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27E81A4FE4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:54:44.055579079Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:54:44.07489233Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:54:44.074931464Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:54:44.074950808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:44.074959991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:54:44.07504266Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-06-29T23:54:44.075050343Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:54:44.075070497Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.394µs success=true
level=info ts=2026-06-29T23:54:44.372283667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:54:44.375728572Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27E9CC998EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:54:47.051127612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:54:47.054884779Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27F3C770172, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:54:48.548443844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:54:48.551741202Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB27F95B2D9A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:54:49.075291069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:51.358379264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:54:51.362404642Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2803D325784, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:54:52.658096719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:54:52.66147612Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2808AA9458A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:54:53.084534327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:54:53.087883091Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB280A41331FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-06-29T23:54:54.074412028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:54:59.074733119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:54:59.431592759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:54:59.435559345Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2821E6953E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:54:59.435579747Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:54:59.435610639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:54:59.438863316Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2821E9CE2ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:55:00.002233366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:55:00.005664213Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2824068A05A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:55:00.056122451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:55:00.060100168Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28243A738C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:55:04.075218529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:04.270162208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:55:04.27717125Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2833F02654E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-29T23:55:06.985841993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:55:06.989147731Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB283E0A88830, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:55:07.306565824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:55:07.309777156Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB283F3C3FFE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:55:07.836088631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:55:07.844661457Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28413A4A46E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:55:07.925978513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:55:07.931951102Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28418D9D277, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:55:08.279049276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:55:08.282400355Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2842DBCDE21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:55:09.009290508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:55:09.01402363Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2845946D0D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:55:09.074605881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:14.074481624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:55:14.074481676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:14.074512883Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:55:14.816457157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:55:14.819788348Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB285B36570CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:55:16.792693333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:55:16.796282514Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2862933D3AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:55:16.796297045Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:55:16.79631482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:55:16.799116029Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28629616066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:55:16.98569415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:55:16.989553287Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28634B93403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:55:17.566668332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:55:17.574354731Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28657958A0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:55:17.574374634Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:55:17.574397322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:55:17.581114723Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28657FDB9A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:55:17.759432639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:55:17.766215781Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28662FD70FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:55:17.861560391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:55:17.865344262Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28668E7941B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:55:17.865361053Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:55:17.874475539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:55:17.878041358Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28669A97436, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:55:18.89177108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:55:18.896026219Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB286A65AC5AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:55:19.075036521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:21.925382358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:55:21.931466037Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2875B49CE12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:55:23.802082021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:55:23.805326719Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB287CAFA1BAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:55:24.075027378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:55:29.075057153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:30.464826925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:55:30.468212182Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB289581DDF28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:55:31.303546868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:55:31.306984665Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2898A1B67C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:55:33.784847362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:55:33.790319095Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28A1E21FAB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:55:34.074361494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:34.432196416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:55:34.439148733Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28A44CA32DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:55:35.952484153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:55:35.957682341Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28A9F4FFA88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:55:37.666048666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:55:37.671858939Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28B057C49DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:55:39.075012774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:39.402884957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:55:39.40620645Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28B6CDD0029, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:55:41.237676641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:55:41.241264767Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28BDA3DDA7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:55:41.465094686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:55:41.471812899Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28BE7F6D9E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:55:44.05651892Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:55:44.074945624Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:55:44.074981263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:55:44.074975761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:44.161476889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:55:44.164801003Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28C887F8897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:55:46.158486831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:55:46.161864888Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28CFF87F408, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:55:48.345383059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:55:48.348727229Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28D81DFE4F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-06-29T23:55:49.074369435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:49.186746632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:55:49.191315121Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28DB4098D7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:55:49.843570713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:55:49.847070701Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28DDB2BEDD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:55:49.8470868Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:55:49.847105431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:55:49.849906551Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28DDB5C90D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:55:50.295403831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:55:50.29907563Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28DF61CC20B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:55:50.902818576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:55:50.906022368Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28E1A4E8A65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:55:53.850920807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="909 B" avg_comp="454 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:55:53.854184698Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28ECA069852, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:55:53.854203831Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:55:53.85422381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:55:53.856944221Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28ECA32F9A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:55:54.074496069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:55:55.115259691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:55:55.118727204Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28F156572AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:55:56.188012379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:55:56.191275714Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28F5553BB57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:55:58.421522514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:55:58.425203866Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB28FDA77BD02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:55:59.075240485Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:00.512201039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:56:00.515921067Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29057189F12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:56:01.108755915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:56:01.115800957Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2907ADA0537, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:56:01.986690353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:56:01.989990315Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB290AEF5CA57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:56:03.609470584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:56:03.879356777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:56:03.884932051Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2911FE85C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:56:04.075308483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:04.11942357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:56:04.124259916Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2912E28668F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:56:05.564692712Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2918406F1AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:56:05.782652104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:56:05.788460999Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB291915E6F7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:56:06.265969536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:56:06.270804677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB291AE1EDF04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:56:06.336856802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:56:06.342758258Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB291B2674B27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:56:06.858206911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:56:06.861488214Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB291D1532D8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:56:09.075299658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:09.298366043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:56:09.302296305Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29262CA6FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:56:14.074481266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:56:14.074511777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:56:14.0745068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:56:19.074513083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:19.38161876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:56:19.385317555Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB294BBCAFE3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:56:19.549965504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:56:19.553194327Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB294C5CF46EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:56:19.553210916Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:56:19.553228405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:56:19.556073576Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB294C5FD52C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:56:22.719463524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:56:22.723352774Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29582C361CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:56:24.074653905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:24.822285682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:56:24.829436531Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB296004D734E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:56:25.718679246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:56:25.724475765Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29635A4031A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:56:27.113577963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:56:27.119366546Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29688CA5179, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:56:27.860432177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:56:27.863721544Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB296B527399F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:56:28.198393849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:56:28.202692251Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB296C94CA36D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-06-29T23:56:29.075203891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:31.87497142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:56:31.878416282Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB297A47354FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:56:34.075285877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:34.899758299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:56:34.903011883Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29858BBECD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:56:37.57401699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:56:37.577777264Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB298F824EB55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:56:38.576287958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:56:38.579829509Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29933DFF6E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-29T23:56:39.075373175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:39.672072591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:56:39.675324759Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB299752E3299, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:56:41.271193073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:56:41.274451585Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB299D480603B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-06-29T23:56:41.274468544Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:56:41.274492368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:56:41.277183953Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB299D4AC087F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:56:42.114769766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:56:42.11809748Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29A06C91068, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-06-29T23:56:42.898307945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:56:42.901664486Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29A357C05E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:56:44.05564321Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:56:44.074850979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:56:44.074879973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:44.074892629Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:56:44.798676836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:56:44.802289383Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29AA6C353E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:56:45.201678147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:56:45.20608395Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29ABECA1CF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:56:48.516638356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:56:48.520273945Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29B84619274, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:56:48.71977223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:56:48.725800809Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29B90A202FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:56:48.777988313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:56:48.782034496Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29B93FC5AE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:56:48.985487011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:56:48.991849996Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29BA07A4F65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:56:49.074930525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:51.210754228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:56:51.215800004Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29C250B4958, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:56:51.577811034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:56:51.583900845Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29C3AFBB941, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:56:51.75736617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:56:51.765307562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29C45CDA60E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:56:53.063035751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:56:53.068904035Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29C937F6E33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-06-29T23:56:54.074849786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:56:56.254914892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:56:56.258378743Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29D519B44D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:56:56.258398877Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:56:56.258419914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:56:56.261158429Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29D51C9185F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:56:56.418486686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:56:56.421773513Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29D5B594D90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:56:56.603222185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:56:56.606511531Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29D665C9911, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:56:59.075305724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:57:04.074582081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:07.377753804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:57:07.383013421Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB29FE8AF4997, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:57:08.429115914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:57:08.432320788Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A0273C7BB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:57:08.432338074Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:08.432358591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:57:08.435183586Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A0276A0074, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:57:09.074944794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:10.970583923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:57:10.973966361Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A0BEBB2F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:57:11.46482908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:57:11.468200541Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A0DC2ED4EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:57:11.468223937Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:11.468251893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:57:11.471001302Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A0DC5CF671, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:57:13.625680304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:57:13.629062016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A15CFB73EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:57:14.07499479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:57:14.075040264Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:57:14.075051072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:17.979831611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:57:17.983092215Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A260812BB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:57:17.983110142Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:17.98312973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:57:17.985778265Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A260ACFCBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:57:18.864526712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:57:18.868246736Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A2953E9157, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:57:18.868267747Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:18.868289814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:57:18.871589788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A295739993, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:57:19.075166034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:19.080319984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:57:19.085685384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A2A2393CB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:57:20.788280314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:57:20.796373913Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A308309F9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-29T23:57:24.074826479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:24.602510145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:57:24.608638444Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A3EB69510E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:57:26.611393301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:57:26.614809431Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A462FD65DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:57:26.713160787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:57:26.71643323Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A4690D6729, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:57:27.512834128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:57:27.516255952Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A498B8F116, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:57:27.885050372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:57:27.888556252Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A4AEE96A0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:57:28.174670064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:57:28.178122911Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A4C02B1AC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:57:28.673668485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:57:28.677436989Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A4DDE97106, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:57:28.913178635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:57:28.91685394Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A4EC317907, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:57:29.074345991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:29.602287684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:57:29.607344388Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A5154913B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:57:32.153799997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:57:32.157798455Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A5AD5A81F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:57:33.518516628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:57:33.522682483Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A5FEB469A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:57:33.771458949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:57:33.775373711Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A60DC50C73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:57:34.074535557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:35.708172327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:57:35.716438884Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A6817D252B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:57:37.808844516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:57:37.814921621Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A6FE91057D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-06-29T23:57:37.81494454Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:37.814967868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:57:37.817756825Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A6FEBED10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:57:37.851964064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:57:37.854939054Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A700F4E30F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:57:37.965337219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:57:37.969509835Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A707C68A75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-29T23:57:39.075103231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:40.343792706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:57:40.347148925Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A7957F9B52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-29T23:57:43.770195069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:57:43.77445023Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A861C91CC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:57:44.055666147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:57:44.074651177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:57:44.074677199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:57:44.074680362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:44.898648463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:57:44.90528358Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A8A5303E5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:57:46.857830007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:57:46.865595331Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A91A090E67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:57:46.920775068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:57:46.926812079Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A91DAE0272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:57:46.92682819Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:46.926848651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:57:46.929705618Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A91DDD3533, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:57:47.302449721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:57:47.305902078Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A93446E9F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:57:47.305918402Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:47.305939197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:57:47.308769774Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A934745EEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:57:47.795381123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:57:47.798762556Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A951A7545F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:57:47.798782593Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:57:47.798808066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:57:47.801638787Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A951D571AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:57:48.816578293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:57:48.820044226Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A98E878BBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:57:49.075123364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:49.974670142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:57:49.978012784Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A9D38CB1E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:57:50.620770697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:57:50.624191105Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2A9FA0ED22E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-29T23:57:53.517747894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:57:53.521159775Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AAA6BCCD26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:57:54.07527115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:55.60749045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:57:55.6108426Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AB234A91EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:57:58.305990244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:57:58.309490811Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ABC4231B45, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-29T23:57:59.075259045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:57:59.704179629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:57:59.707906792Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AC177C427C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:58:00.984085295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:58:00.987665999Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AC63C50AA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:58:02.385680675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:58:02.389181135Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ACB74F05D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:58:04.074497619Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:07.425716718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:58:07.431212643Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ADE3D65E5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:58:08.383742808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:58:08.391048554Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AE1D0DD61F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:58:08.391068733Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:58:08.391091168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:58:08.393921615Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AE1D3B18A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:58:08.43800916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:58:08.441613436Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AE200C7459, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-29T23:58:09.074675177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:09.275376238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:58:09.278886016Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AE51F95600, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:58:09.439198769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:58:09.44499378Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AE5BDE8DA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:58:10.768734912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:58:10.773677855Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2AEAAFE8E29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-06-29T23:58:14.074350592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:14.074378391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:58:14.074395873Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:58:17.124150496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:58:17.128200745Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B025CC9F00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:58:18.244649864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:58:18.248612006Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B06894C6B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:58:18.909579764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:58:18.912936971Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B090342E13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-29T23:58:19.075161039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:19.638440151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:58:19.646517507Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B0BBED4224, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:58:20.140877468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:58:20.148432055Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B0D9D9087E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:58:20.148453303Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:58:20.148474871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:58:20.151210095Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B0DA056983, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:58:24.074527998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:25.396219514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:58:25.399938645Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B212D977ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:58:25.441006667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:58:25.444534696Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B21585261C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:58:27.758505933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:58:27.764514348Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B29FCBE64F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:58:28.879674382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:58:28.882883247Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B2E275E34A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-06-29T23:58:29.075176155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:30.143681853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:58:30.147075819Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B32DCE9A9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:58:30.969104043Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2B35ED10C74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-06-29T23:58:31.039133516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:58:31.042360386Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B3632CB816, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:58:31.842777894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:58:31.846173338Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B393154789, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:58:31.953668081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:58:31.958712902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B399C82E4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:58:32.335737519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:58:32.341417705Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B3B09AB07D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:58:32.346435509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:58:32.349448453Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B3B11546A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-29T23:58:33.571079041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:58:33.575395642Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B3FA26B7CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:58:33.57541664Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:58:33.575442243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:58:33.582579912Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B3FA926DE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:58:33.908407073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:58:33.914116748Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B40E590FB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:58:34.074423222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:35.313662771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:58:35.317016089Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B461F6806E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-29T23:58:39.075156392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:39.228466302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:58:39.231867872Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B54B4F6218, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:58:43.82768153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:58:43.831088386Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B65D7082ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-06-29T23:58:44.055617504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:58:44.075027653Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-29T23:58:44.075037737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:44.075058492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:58:46.775692976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:58:46.779111348Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B70D287058, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-29T23:58:49.074695649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:50.704761943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:58:50.708226266Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B7F759F46C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-29T23:58:51.302750888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:58:51.307508682Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B81B0B6A8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-29T23:58:54.075217484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:58:54.756050397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:58:54.763440094Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B8E90D4259, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:58:56.298763783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:58:56.30441385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B944E74350, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:58:56.304438842Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:58:56.304463866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:58:56.312391846Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B94562AE3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-29T23:58:57.028397683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:58:57.034350958Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B9706B4982, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:58:57.708383825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:58:57.711749838Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2B998CA7CD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=warn ts=2026-06-29T23:58:59.074746172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-29T23:59:04.075039237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:04.14219262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:59:04.146107753Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BB184ABF5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:59:05.244151891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:59:05.247804603Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BB59F9C01E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-06-29T23:59:05.247823454Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:59:05.247844271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:59:05.253571937Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BB5A533856, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:59:05.367146035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:59:05.372962101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BB616FF4D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-29T23:59:06.882205759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:59:06.886863374Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BBBB9AB73E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:59:07.775686694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:59:07.779468398Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BBF0DC7DB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-29T23:59:09.075003197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:09.122149078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-29T23:59:09.125385931Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BC411A90AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-29T23:59:11.678018083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:59:11.681324623Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BCD971AC02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:59:12.429126198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-29T23:59:12.433991349Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BD064E22C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-29T23:59:13.609240841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-29T23:59:13.614849718Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BD4CB1BF81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-29T23:59:14.075126059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:59:14.075156421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-29T23:59:14.075162374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:14.081359377Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-06-29T23:59:14.083889797Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008704041s
level=info ts=2026-06-29T23:59:15.492762698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:59:15.500716573Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BDBD1939D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:59:15.500735998Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:59:15.500762531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:59:15.510676302Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BDBDB1C3F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:59:15.544754835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-29T23:59:15.550493114Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BDC01102C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-29T23:59:15.752997883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-29T23:59:15.756937193Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BDCC5895D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-29T23:59:16.038585573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:59:16.044107736Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BDDD7C8891, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-29T23:59:19.074769216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:19.203935467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-29T23:59:19.210035843Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BE9A2EE628, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-29T23:59:19.606078803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-29T23:59:19.609833608Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BEB20199FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-29T23:59:22.803446593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:59:22.807282292Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BF709A3C2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:59:22.807306682Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:59:22.807334274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:59:22.813097421Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BF70F3A8B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:59:24.066558938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-29T23:59:24.073020828Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BFBC0C8DA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-29T23:59:24.075098142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:24.715353869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-29T23:59:24.719004866Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BFE28D858A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-29T23:59:24.836258806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-29T23:59:24.843171859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2BFE9F21E75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-29T23:59:27.676591204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-29T23:59:27.687680939Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C0937F11D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:59:28.113678469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-29T23:59:28.119361214Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C0AD39B268, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-29T23:59:29.074705037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:29.412971923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-29T23:59:29.416317451Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C0FA8772F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-29T23:59:30.397688313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-29T23:59:30.401258716Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C1353B97AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-06-29T23:59:34.074764725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:38.180286313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-29T23:59:38.183793218Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C3051BB018, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-29T23:59:39.075141113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:41.958671468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-29T23:59:41.962195997Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C3E6517C19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-29T23:59:42.406401363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-29T23:59:42.410328236Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C401093314, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-29T23:59:43.331244413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-29T23:59:43.33822739Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C438584152, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-29T23:59:44.056568326Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-29T23:59:44.074254017Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-06-29T23:59:44.074378225Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-06-29T23:59:44.074387001Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-06-29T23:59:44.074404121Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.284µs success=true
level=warn ts=2026-06-29T23:59:44.074475626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:44.07449789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-29T23:59:44.074524207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-29T23:59:45.776742143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-29T23:59:45.780168429Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C4C9E40F1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-29T23:59:49.074705475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:49.708743838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-29T23:59:49.712519219Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C5B443CB76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-29T23:59:51.878261424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:59:51.881694724Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C635916DFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-06-29T23:59:51.881713189Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:59:51.881731506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-29T23:59:51.884345726Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C635BE163E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-29T23:59:52.278453625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-29T23:59:52.282184026Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C64D6D0780, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-29T23:59:52.772592465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-29T23:59:52.776072299Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C66AE21EE5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-29T23:59:52.948507228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-29T23:59:52.951722409Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C6755A4111, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-29T23:59:53.184325602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:59:53.188812877Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C6836CF4C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:59:53.188831947Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-29T23:59:53.188854797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-29T23:59:53.192191176Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C683AB52B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-29T23:59:53.501040471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-06-29T23:59:54.0752225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-29T23:59:54.56166852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:59:54.570954909Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C6D5DC493E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-06-29T23:59:54.773584577Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C6E1F1CB46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-29T23:59:55.76447611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-29T23:59:55.770171416Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C71D582D2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-29T23:59:59.075018193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:00.240895687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T00:00:00.244412538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C82807ABF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T00:00:00.348101783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:00:00.351512592Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C82E6742D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T00:00:04.06298838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T00:00:04.066604891Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C90BD89207, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T00:00:04.074683624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:04.310669443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:00:04.317165341Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C91AC2AB1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T00:00:04.36836912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:00:04.373790441Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C91E25891C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T00:00:04.89769733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-30T00:00:04.905137799Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C93DC42305, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-30T00:00:04.92416772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:00:04.92728781Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C93F272CBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T00:00:05.160918802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T00:00:05.165317302Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C94D53C0ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T00:00:05.676516855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T00:00:05.682308289Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C96C259E72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T00:00:07.782112956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:00:07.785575196Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2C9E9828285, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T00:00:09.074757506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:11.028300621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T00:00:11.031829344Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CAAB013C41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T00:00:14.074870843Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:00:14.074903224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T00:00:14.074927333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:15.082225892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-30T00:00:15.085762336Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CB9CA283C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-30T00:00:15.706058487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:00:15.709877803Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CBC1D3552D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T00:00:17.243542095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T00:00:17.247761036Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CC1D8173FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-06-30T00:00:19.074388215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:22.300285144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T00:00:22.309278231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CD4B300B73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T00:00:22.955461399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T00:00:22.95898314Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CD71EA61CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T00:00:24.075210026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:25.531940023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:00:25.535381165Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CE0B7BB9AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T00:00:25.866456843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T00:00:25.870067147Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CE1F6CC1B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T00:00:29.074690865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:29.906894115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:00:29.910873619Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2CF10433728, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T00:00:34.074861612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:34.93824686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T00:00:34.941640424Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D03C23E1B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T00:00:35.17456971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-30T00:00:35.178244376Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D04A3D262E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-30T00:00:38.290185468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T00:00:38.29436503Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D103F60D73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-06-30T00:00:39.075387935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:39.208967464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="909 B" avg_comp="454 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:00:39.212507934Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D13AB3D5A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T00:00:39.659113802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T00:00:39.66242015Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D15584FFAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T00:00:39.720431495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T00:00:39.723805105Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D1592E0F03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T00:00:40.705841278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:00:40.709260367Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D193EA9A24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T00:00:42.800122891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="852 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T00:00:42.805880156Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D210E32BC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T00:00:43.647245158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-30T00:00:43.654768594Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D24369F8C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-30T00:00:44.055650029Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-06-30T00:00:44.074620281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:44.074641457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:00:44.074690002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T00:00:45.324061057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T00:00:45.32776853Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D2A73136C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T00:00:46.5500315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T00:00:46.553487031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D2F0427B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T00:00:47.705302446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:00:47.709652009Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D3352643B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T00:00:48.75954175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T00:00:48.760606792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:00:48.764020143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D37403E3D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:00:48.769976436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D3745FBE4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-06-30T00:00:49.075237314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:50.053106723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T00:00:50.059848551Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D3C1414067, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T00:00:52.05633746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:00:52.064590331Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D438BECA3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-06-30T00:00:54.07520065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:00:54.188452138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T00:00:54.194781769Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D4B7B78A4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T00:00:55.120120795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-30T00:00:55.123545666Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D4EF14641C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-30T00:00:55.525600911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:00:55.529203896Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D5074198D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-06-30T00:00:59.074508868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:00.899030634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T00:01:00.903024728Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D6478F1DCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T00:01:01.806654325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:01:01.813719679Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D67DD194B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T00:01:04.074755529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T00:01:09.075383171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:09.778977689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.6 kB" avg_comp="4.6 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T00:01:09.782586085Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D858D3132D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T00:01:09.961373971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T00:01:09.965746345Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D863BEB91C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T00:01:11.972667356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T00:01:11.979293085Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D8DBC2CCA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T00:01:14.074781265Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:01:14.074814007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T00:01:14.07482296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:15.619674794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T00:01:15.623066781Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2D9B4F392A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T00:01:19.074596693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:19.137731545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:01:19.141538339Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DA86A6F166, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:01:19.141558309Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:01:19.141579875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:01:19.144889456Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DA86DAC689, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T00:01:19.573847879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:01:19.57724898Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DAA0A305BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:01:19.577266321Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:01:19.577290518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:01:19.580153389Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DAA0D1B5C4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T00:01:21.512663126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T00:01:21.516719921Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DB14369D88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T00:01:21.531733557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:01:21.535119458Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DB1554DA22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T00:01:24.074688667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:28.087508147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T00:01:28.09090768Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DC9C16F1A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-06-30T00:01:29.075093111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:29.141257301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T00:01:29.14470647Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DCDAE6A0EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T00:01:29.471273717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T00:01:29.47461332Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DCEE9139C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-06-30T00:01:30.113676787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:01:30.117402693Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD14DC9129, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T00:01:31.114408287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T00:01:31.117918875Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD508350D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T00:01:31.560328779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:01:31.56377098Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD6B15D0EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T00:01:31.842575229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:01:31.845919367Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD7BE94647, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T00:01:32.072833239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-30T00:01:32.076180764Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD89A2BACA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-30T00:01:32.41711117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:01:32.420650971Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DD9E28B665, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T00:01:33.284320742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T00:01:33.287808034Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DDD1D89FB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T00:01:33.565691177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="862 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T00:01:33.571465704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DDE2C16530, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T00:01:34.075241814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:36.360582006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-30T00:01:36.368450286Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DE8966E9C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-30T00:01:37.256877302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:01:37.260824356Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DEBEA3A041, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:01:37.260844196Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:01:37.260866309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:01:37.264430558Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DEBEDBDAFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T00:01:38.439522749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T00:01:38.443494301Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DF05267A67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T00:01:39.074590725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:40.411237426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:01:40.419037352Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DF7AE73F04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:01:40.419059054Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:01:40.419083376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:01:40.425733289Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DF7B4F0750, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T00:01:40.698687286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-30T00:01:40.705258825Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2DF8BF062C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-30T00:01:44.055605099Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T00:01:44.074790204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:01:44.074826503Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T00:01:44.074823368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:44.940703619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T00:01:44.948949112Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E088E6E429, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T00:01:48.886834409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T00:01:48.893201241Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E173FFDB98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-06-30T00:01:49.074779635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:50.380138343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T00:01:50.38457005Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E1CCE341E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T00:01:50.511845293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:01:50.519004005Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E1D4E19F6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T00:01:53.9926927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:01:53.996917724Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E2A42E2FA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-06-30T00:01:54.075279574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:01:54.244592428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:01:54.248254983Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E2B32F2117, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T00:01:55.519425991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:01:55.523025775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E2FF2A82C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:01:55.523046472Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:01:55.523071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:01:55.526138446Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E2FF5D1CDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T00:01:55.563326344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:01:55.568730329Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E301E4F4C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T00:01:56.781757492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T00:01:56.788548218Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E34A9860BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-06-30T00:01:59.074604879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:01.997198847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T00:02:02.003235337Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E4816BB916, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-06-30T00:02:04.075362065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:04.673189352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T00:02:04.676823692Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E520C70BBA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-06-30T00:02:07.158067013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T00:02:07.162401993Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E5B4E7708D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T00:02:07.374559767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:02:07.378135762Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E5C1C981AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T00:02:08.276048439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:02:08.27967497Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E5F785B458, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:02:08.279691443Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:02:08.279712785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:02:08.282824652Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E5F7B82F9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T00:02:08.615676095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T00:02:08.619238963Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E60BC41D56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T00:02:09.075352371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:12.014673537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:02:12.018128591Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E6D65A4D15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T00:02:12.537577007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-30T00:02:12.541191966Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E6F587FE7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-30T00:02:14.07440036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:02:14.074444175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T00:02:14.074453254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:14.945108091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="687 B" avg_comp="687 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T00:02:14.948621743Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E78505A84B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T00:02:15.509364429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:15.513030124Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E7A6A96F22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:15.51304744Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:02:15.513066865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:15.516168684Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E7A6DBD73E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-06-30T00:02:15.549323751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T00:02:15.552787996Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E7A9093A66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T00:02:17.508348482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:02:17.513860975Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E81DECE096, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T00:02:18.46880633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="862 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T00:02:18.47818405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E857669211, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-06-30T00:02:19.074397629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:21.15269322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T00:02:21.158619213Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2E8F72AD624, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-06-30T00:02:24.0744105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:25.671981997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T00:02:25.675550613Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EA04653A76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T00:02:26.67036639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:02:26.673798441Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EA3FE6DBF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-06-30T00:02:26.997673075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-06-30T00:02:27.001026958Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EA53670A48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-06-30T00:02:27.291980891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-30T00:02:27.296584535Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EA64F74FDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-06-30T00:02:28.556085847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:02:28.559576808Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EAB04D8A51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-06-30T00:02:29.075353184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:29.31440142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T00:02:29.318146268Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EADD83E417, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-06-30T00:02:29.559216046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="1.4 MB" avg_uncomp="1.4 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-06-30T00:02:29.565878764Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EAEC44E417, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-06-30T00:02:32.274566599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="629 kB" avg_comp="629 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-06-30T00:02:32.27892394Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EB8DFDC189, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-06-30T00:02:33.061884491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T00:02:33.072711696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EBBD4CF6CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T00:02:33.42747955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:33.433566682Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EBD2D051F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:33.723054554Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BDB2EBE414768C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-06-30T00:02:34.075314849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:36.094814166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-06-30T00:02:36.098008159Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EC71A184AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-06-30T00:02:37.141837814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="49 kB" avg_uncomp="49 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-06-30T00:02:37.145081539Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ECB00A5F6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-06-30T00:02:37.164253954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:02:37.167665298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ECB1605275, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:02:37.167679267Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:02:37.167695694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:02:37.17053296Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ECB1901268, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-06-30T00:02:39.074816376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:40.502257184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="320 kB" avg_comp="320 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" full=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-06-30T00:02:40.505952988Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2ED785E0E40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-06-30T00:02:44.056088483Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-06-30T00:02:44.074395897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-06-30T00:02:44.074398342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:44.074430649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-06-30T00:02:44.698893113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="689 kB" avg_uncomp="689 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-06-30T00:02:44.706052117Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EE72B2967E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-06-30T00:02:46.035330333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-06-30T00:02:46.038856376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EEC2262357, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-06-30T00:02:47.862598576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.7 kB" avg_comp="4.7 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-06-30T00:02:47.866014941Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2EF2F0EB9E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-06-30T00:02:49.074611065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-06-30T00:02:54.074568291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:54.623437229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="655 kB" avg_uncomp="655 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-06-30T00:02:54.627103697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F0C20A2F03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-06-30T00:02:54.780589829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="113 kB" avg_uncomp="113 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-06-30T00:02:54.783926067Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F0CB660B0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-06-30T00:02:56.409686677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="72 kB" avg_uncomp="72 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-06-30T00:02:56.413057469Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F12C8075DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-06-30T00:02:58.398414223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="156 kB" avg_comp="156 kB" total_uncomp="774 kB" avg_uncomp="774 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-06-30T00:02:58.402410804Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F1A31026EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-06-30T00:02:58.502837506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="26 kB" avg_comp="26 kB" total_uncomp="131 kB" avg_uncomp="131 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-06-30T00:02:58.50638099Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F1A945EA10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-06-30T00:02:59.074383238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:02:59.125544562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="669 B" avg_comp="669 B" total_uncomp="2.6 kB" avg_uncomp="2.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-06-30T00:02:59.128974466Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F1CE6224CE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-06-30T00:03:01.799017952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=973acd4420a50684 immediate=false num_chunks=1 total_comp="23 kB" avg_comp="23 kB" total_uncomp="223 kB" avg_uncomp="223 kB" full=1 labels="{stream=\"openshift-cluster-samples-operator\"}"
level=error ts=2026-06-30T00:03:01.802347061Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F26DB9D684, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-samples-operator\"}"
level=info ts=2026-06-30T00:03:03.345613959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-06-30T00:03:03.348986089Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F2C9E95907, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-06-30T00:03:04.075096606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:03:04.324015806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="39 kB" avg_comp="39 kB" total_uncomp="262 kB" avg_uncomp="262 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-06-30T00:03:04.327260704Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F3043A1E4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-06-30T00:03:05.269323534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="7.0 MB" avg_uncomp="7.0 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-06-30T00:03:05.274361089Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F33CAD0472, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-06-30T00:03:05.311400135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=2 total_comp="927 B" avg_comp="463 B" total_uncomp="3.6 kB" avg_uncomp="1.8 kB" full=2 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-06-30T00:03:05.318986645Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F33F5543B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-06-30T00:03:09.074988077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:03:09.997077059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="1.7 MB" avg_comp="862 kB" total_uncomp="14 MB" avg_uncomp="7.3 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-06-30T00:03:10.002674186Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F456810356, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-06-30T00:03:11.377459825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="208 kB" avg_uncomp="208 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-06-30T00:03:11.383335501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F4A8CB6C2A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-06-30T00:03:12.284212263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:03:12.287590252Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F4DEB13CE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:03:12.287619936Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:03:12.287665635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="3.7 kB" avg_comp="1.9 kB" total_uncomp="24 kB" avg_uncomp="12 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-06-30T00:03:12.290496206Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F4DEE07462, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-06-30T00:03:14.074340019Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-06-30T00:03:14.074375856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-06-30T00:03:14.074384803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:03:14.565059952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="132 MB" avg_uncomp="33 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-06-30T00:03:14.573140907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F566ECF48D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-06-30T00:03:18.088986383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="304 kB" avg_comp="304 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-06-30T00:03:18.096044217Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F638E5AE4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-06-30T00:03:19.074698292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-06-30T00:03:20.89544122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="16 kB" avg_comp="16 kB" total_uncomp="149 kB" avg_uncomp="149 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-06-30T00:03:20.901460083Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F6E01DC9DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-06-30T00:03:20.901485665Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-06-30T00:03:20.901517323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.3 kB" avg_comp="1.3 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-06-30T00:03:20.904560562Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F6E0507168, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-06-30T00:03:21.98658933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="177 kB" avg_comp="177 kB" total_uncomp="1.1 MB" avg_uncomp="1.1 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-06-30T00:03:21.99122031Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BDB2F721034AD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"