level=info ts=2026-07-02T22:08:57.990794327Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-07-02T22:08:57.990846632Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=info ts=2026-07-02T22:08:57.992862968Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=info ts=2026-07-02T22:08:57.994040925Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-44778b07
level=info ts=2026-07-02T22:08:57.994334351Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-07-02T22:08:57.998027412Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
ts=2026-07-02T22:08:58.000030889Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-07-02T22:08:58.000127018Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T22:08:58.000167471Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=5.814614ms
level=error ts=2026-07-02T22:08:58.00018209Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-07-02T22:08:58.000190626Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-07-02T22:08:58.000135983Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-07-02T22:08:58.00018779Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:08:58.000832523Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=info ts=2026-07-02T22:08:58.000861046Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-07-02T22:08:58.000881175Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
ts=2026-07-02T22:08:58.002921929Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T22:08:58.002939302Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-07-02T22:08:58.017592675Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-07-02T22:08:58.017650863Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-07-02T22:08:58.017671883Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=info ts=2026-07-02T22:08:58.017993681Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-07-02T22:08:58.017993292Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-07-02T22:08:58.018012052Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=info ts=2026-07-02T22:08:58.01805319Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-07-02T22:08:58.018190479Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-07-02T22:08:58.018262405Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-07-02T22:08:58.018270248Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:08:58.018322408Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=23.713µs success=true
level=info ts=2026-07-02T22:08:58.018336168Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-07-02T22:08:58.018430573Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=167.827µs errors=false
level=info ts=2026-07-02T22:08:58.018444801Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-07-02T22:08:58.018591286Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=331.065µs errors=false
level=info ts=2026-07-02T22:08:58.018602079Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-07-02T22:08:58.018607275Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=347.129µs
level=info ts=2026-07-02T22:08:58.018660448Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-07-02T22:08:58.018662209Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-07-02T22:08:58.018684022Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-07-02T22:08:58.018682554Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=2.014135889s
level=info ts=2026-07-02T22:08:58.018772703Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-07-02T22:08:58.018783849Z caller=loki.go:615 msg="Loki started" startup_time=91.973926ms
level=info ts=2026-07-02T22:08:58.018787263Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
ts=2026-07-02T22:08:59.56300098Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T22:08:59.563028502Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-07-02T22:09:02.997073095Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T22:09:02.997092043Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T22:09:03.01944809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-07-02T22:09:07.183399001Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-07-02T22:09:07.183421292Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-07-02T22:09:08.019214529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:13.018975495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:09:15.956939935Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=17.956744187s
level=warn ts=2026-07-02T22:09:18.01945858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:23.019201731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:09:28.019104126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:09:28.019131671Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-07-02T22:09:28.019140816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:09:28.019197271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:33.019331171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:38.019777787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:43.019094976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:48.018992296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:09:53.01897553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:09:58.000284138Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:09:58.018847248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:09:58.018876007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:09:58.018876524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:03.019262778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:08.018950963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:10:12.261097277Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9887F992872E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:10:13.019421773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:18.019519879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:23.019476564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:10:28.019460967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:10:28.019494887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:10:28.019498352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:33.019571653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:10:36.8852572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:10:36.89759332Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE988DB602FE52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:10:38.018809964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:43.019208255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:10:48.018901369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:10:52.967669634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:10:52.979816542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98917493A8D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:10:53.019820097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:10:58.000246697Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:10:58.019619074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:10:58.01964578Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:10:58.019645543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:03.019053075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:08.01967462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:13.019398646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:18.018945179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:23.019566174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:11:23.411536133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:11:23.423759051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98988B2DC210, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:11:28.018803091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:11:28.018821052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:11:28.0188463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:11:33.019580525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:38.019342865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:43.019032469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:48.018870715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:11:53.019643219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:11:58.000225819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:11:58.018755332Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:11:58.018791192Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:11:58.018817363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:03.019226517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:08.019051984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:13.019105392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:12:15.906710945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:12:15.918262176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98A4C418AD77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:12:18.019132996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:23.019135923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:12:28.019573396Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:12:28.019568397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:12:28.01960815Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:12:33.019855532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:38.019849677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:43.019860259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:48.019234195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:12:53.019395457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:12:58.000241483Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:12:58.018806076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:12:58.018831509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:12:58.018854925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:13:03.019035769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:08.019683211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:13:10.264901245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:13:10.381356785Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98B1725ABAE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:13:13.019021645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:18.018842278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:23.01942734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:28.019091771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:13:28.019109294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:13:28.01914573Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:13:33.019507817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:38.018815548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:43.018931476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:48.019370513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:13:53.01985783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:13:58.000243004Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:13:58.01965019Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T22:13:58.019675856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:13:58.019683656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:13:58.019704932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:13:58.019753817Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-02T22:13:58.019762096Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:13:58.019775898Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.16µs success=true
level=info ts=2026-07-02T22:13:58.233252418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:13:58.243536473Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98BC9729A623, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:14:03.019292503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:14:04.479296499Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE98BE0ADAF6F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:14:08.019753212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:13.019820896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:18.019042236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:23.019716517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:14:28.018815106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:14:28.018847323Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:14:28.018838592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:33.019494687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:38.01925744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:43.019762654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:14:47.909729902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:14:47.921292222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98C8282FD0F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:14:48.019253662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:14:53.019810514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:14:55.110082873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:14:55.116769679Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98C9D5136ACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:14:58.000841085Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:14:58.019074495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:14:58.019079145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:14:58.019101307Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:15:03.019817877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:08.019036972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:12.188472848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:15:12.196345255Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98CDCF16DE2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:15:13.019325853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:18.019796039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:23.019682164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:28.019705042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:28.019719629Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:15:28.01974543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:15:33.019833118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:33.507188942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:15:33.515722319Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98D2C5D48763, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:15:38.019656197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:43.019191025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:45.686753822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:15:45.698497323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98D59BFAF4A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:15:48.019245603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:15:53.019668779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:58.001198391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:15:58.019492567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:15:58.019495442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:15:58.019516433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:16:03.019556522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:08.019140155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:13.019669416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:18.018811717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:16:22.867347975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:16:22.875006026Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98DE43DF19CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:16:23.019901743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:16:28.019678512Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:16:28.019707488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:16:28.019688853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:16:28.662530727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:16:28.672246468Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98DF9D6972BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:16:33.01980595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:38.019205478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:43.019831555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:48.019456363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:16:53.019397113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:16:58.000286864Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:16:58.018873941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:16:58.018886027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:16:58.01890908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:17:03.019065357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:03.122340242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:17:03.129554513Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98E7A33A9391, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:17:08.019758072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:12.370322593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:17:12.380404458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98E9CA9FE6F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:17:12.380423776Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:17:12.380445906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:17:12.390247751Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98E9CB386FE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:17:13.019751624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:17:18.019769584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:17:23.019760267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:26.823096699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:17:26.835118102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98ED282EE2A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:17:28.019551728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:17:28.019541072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:28.019591279Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:17:33.019103929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:17:38.019873419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:17:43.019539605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:47.148515813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:17:47.155677722Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98F1E364684D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:17:48.019670927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:17:53.019191528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:55.184807578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:17:55.19003986Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98F3C2485B3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:17:58.000495104Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:17:58.018805461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:17:58.018810951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:17:58.018826494Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-07-02T22:17:59.136396446Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE98F4AD82FA3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:18:03.019495516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:05.894731886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:18:05.905178592Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98F640F2E5B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:18:08.019044895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:18:13.019719115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:13.709108598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:18:13.714583399Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98F8126DBC56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:18:18.019459567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:18:23.019193803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:28.019635715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:18:28.019653288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:28.019672166Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:18:28.026918894Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-07-02T22:18:28.030191142Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010293804s
level=warn ts=2026-07-02T22:18:33.019191616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:35.175782648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:18:35.183538325Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98FD1213E022, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:18:38.019856536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:42.309526023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:18:42.315043314Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE98FEBB25106D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:18:43.019654415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:18:48.019504458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:18:53.019209042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:58.00038357Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:18:58.018759902Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:18:58.018760442Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T22:18:58.018790509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:18:58.018817665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:18:58.018878309Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-02T22:18:58.018889624Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:18:58.018910356Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.458µs success=true
level=info ts=2026-07-02T22:19:00.817036206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:19:00.82674594Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99030A88969C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:19:03.018990679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:08.019270976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:13.019832665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:19:16.009702757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:19:16.017598486Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE990693F8AC49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:19:18.019824606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:23.019005287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:28.019513747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:19:28.019524921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:19:28.019549634Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:19:33.019025518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:19:34.308053711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:19:34.315111651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE990AD695E63B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:19:38.01964746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:43.019879654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:48.019704183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:19:53.019733927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:19:56.004191508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:19:57.095196105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9910246387E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:19:58.000624027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:19:58.019053826Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:19:58.019076083Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:19:58.019091192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:19:59.041190706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:19:59.047399281Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE991098BFB542, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:20:03.019611269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:08.019180208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:13.01980462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:18.018966457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:23.019527998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:20:28.019194422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:20:28.019231068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:20:28.019223995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:20:30.745161029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:20:30.752805412Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9917FA8A3F2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:20:33.019412068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:38.019367355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:20:39.554579291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:20:39.561292672Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE991A078F81BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:20:39.561311832Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:20:39.561332001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:20:39.566735996Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE991A07E72B70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:20:40.615835935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:20:40.625072158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE991A46F8142A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:20:43.018934182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:20:48.019446947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:20:50.870745979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:20:50.877747729Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE991CAA149DD7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:20:53.019084026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:20:58.000688957Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:20:58.01904559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:20:58.019068626Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:20:58.019094878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:03.019182661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:08.019343885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:13.018981514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:18.019025256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:21:20.837972913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:21:20.839040906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:21:20.846364867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9923A45908C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:21:20.847386215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9923A46AC957, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:21:21.876087784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:21:21.882265568Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9923E217D7B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:21:23.019474436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:21:28.018888979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:21:28.018915562Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:21:28.018928096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:33.019834385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:38.019027943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:21:41.255886327Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE992864DCA666, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:21:43.019855857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:48.019249066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:21:53.019074033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:21:58.000415136Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:21:58.018860097Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:21:58.018883756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:21:58.018891269Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:03.019610907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:07.750938586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:22:07.758630883Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE992E9089CB04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:22:08.019696753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:13.019844804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:15.768191675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:22:15.77729755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99306E7E5EBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:22:16.239909472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:22:16.244928698Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99308A5B9A41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:22:18.019843002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:23.018888712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:28.019453151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:22:28.019470813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:28.019486733Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:22:33.018987558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:38.019396603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:43.019022045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:48.019786501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:22:53.019278375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:58.001182691Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:22:58.019638304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:22:58.019669365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:22:58.01967722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:22:58.46245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:22:58.47174353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE993A5F47331E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:23:00.283199883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:23:00.289101567Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE993ACB994453, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:23:03.019823184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:03.660426426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:23:03.665738047Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE993B94DD7E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:23:08.019500409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:13.019768439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:18.019523989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:23.019875863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:28.019401814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:23:28.019432539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:23:28.019429316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:28.024661883Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-07-02T22:23:28.026426612Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007383487s
level=warn ts=2026-07-02T22:23:33.019242918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:38.019459418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:43.019907562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:23:48.019359498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:48.167944273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:23:48.178606993Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9945F207A14C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:23:48.178626797Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:23:48.178647614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:23:48.187101105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9945F28C63E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:23:50.444927501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:23:50.451107899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9946797D5EFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:23:53.019860944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:56.502196135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:23:56.507663905Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9947E27C2282, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:23:58.000379992Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:23:58.018810881Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:23:58.018836823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:23:58.01882936Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T22:23:58.018842613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:23:58.018941609Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-02T22:23:58.018948237Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:23:58.018977341Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=24.495µs success=true
level=info ts=2026-07-02T22:24:01.238891459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:24:01.247809241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9948FD05ABD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:24:03.0190092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:08.019161048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:13.019302976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:18.019017026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:23.01910584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:24:28.019301553Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:24:28.019336702Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:24:28.019342546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:24:29.665569817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:24:29.676419695Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE994F9B7E662E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:24:33.019820731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:24:37.8922568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:24:37.897597898Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE995185842194, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:24:37.897616633Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:24:37.897637574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:24:37.901907797Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE995185C94F7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:24:38.019409244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:43.019429473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:24:48.018864154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:24:49.676000904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:24:49.684951746Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99544417CBE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:24:51.233526087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:24:51.238680656Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9954A0B603BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:24:53.019671903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:24:58.001140976Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:24:58.019576473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:24:58.019610171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:24:58.019628439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:03.019186297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:08.019131876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:12.66519016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:25:12.676204723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99599E7BC2E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:25:13.019461553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:14.381790972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:25:14.386736223Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE995A0470F104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:25:18.019652676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:23.018957354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:28.018817251Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:25:28.018826003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:28.018844428Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:25:31.395759628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:25:31.403346309Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE995DFAB419BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:25:33.019537656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:38.019808386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:43.019413562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:48.018875708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:25:53.019523436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:58.000960441Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:25:58.01939478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:25:58.019420459Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:25:58.01945751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:25:58.554031562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:25:58.564553091Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99644DA41465, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:26:00.812518382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:26:00.817823963Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9964D3F09B26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:26:03.019141892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:26:08.019115392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:26:13.019598581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:26:17.546650281Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9968B913065D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:26:18.019124051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:22.072670691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:26:22.078687709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9969C7320B0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:26:23.019016421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:28.018865097Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:26:28.018863955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:28.018892028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:26:33.01939287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:26:38.019793782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:26:43.018954945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:47.159219476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:26:47.165857716Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE996F9E813DBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:26:48.019745368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:26:53.019684365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:54.058467708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:26:54.067466405Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE997139E1131F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:26:58.000432723Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:26:58.018896998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:26:58.018897249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:26:58.018919966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:27:03.019812747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:27:08.018874635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:27:13.019803137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:27:18.019658305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:21.683547007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:27:21.691790609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9977A8684134, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:27:21.691811185Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:27:21.691832251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:27:21.697345014Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9977A8C066EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:27:23.019752457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:28.019210475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:27:28.019221701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:28.019246156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:27:33.018953163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:36.247480531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:27:36.258938411Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE997B0CACDB23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:27:38.018987536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:39.138408884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:27:39.145032498Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE997BB8B2850A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:27:43.018947003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:45.333406629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:27:45.338795859Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE997D29E1D7AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:27:48.019718968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:27:53.019211279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:27:58.001056619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:27:58.019622732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:27:58.019654626Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:27:58.019648914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:03.019200611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:08.019204286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:13.019350813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:13.191735871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:28:13.200023923Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9983A688F88B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:28:18.018948968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:23.018980723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:28.019662775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:28:28.019673639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:28.019691554Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:28:28.025619868Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-07-02T22:28:28.027378432Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008256008s
level=info ts=2026-07-02T22:28:29.384051925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:28:29.395103638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99876BD749EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:28:33.019338049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:38.019468243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:43.018963968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:43.181042032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:28:43.186750085Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE998AA1E452BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:28:48.019321926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:28:53.019405591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:58.000445803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:28:58.018939505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:28:58.018958215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:28:58.018966506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:28:58.0189889Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T22:28:58.019107829Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-02T22:28:58.019116743Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:28:58.019134075Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.751µs success=true
level=info ts=2026-07-02T22:29:01.06805771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:29:01.075000924Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE998ECC1D2999, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:29:03.019122034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:08.019449053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:13.019474602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:18.018948059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:23.019661552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:29:25.729193024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:29:25.740044351Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99948A44CC98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:29:28.019260981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:29:28.019285807Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:29:28.019298495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:29:31.764077123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:29:31.769474531Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9995F1A54004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:29:33.019402916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:38.019278925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:43.019855503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:29:48.019130385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:29:51.93175546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:29:51.939243551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE999AA3DB4C42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:29:53.019786021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:29:58.001134219Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:29:58.019707686Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:29:58.019709828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:29:58.019733563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:30:03.019825495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:30:08.019447912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:10.228044554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:30:10.239204873Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE999EE69B1273, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:30:10.239231642Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:30:10.239260857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:30:10.248017727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE999EE7277C00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:30:13.019267636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:30:18.019040768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:19.40817766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:30:19.413527638Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99A10972FDD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:30:23.019489312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:30:28.019244702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:28.019250319Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:30:28.019282097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:30:28.128081517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:30:28.138491875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99A3117E3FC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:30:33.019842956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:30:33.637227054Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE99A45942C1F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:30:38.018863407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:30:43.019768164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:45.167839686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:30:45.176512804Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99A7090B46A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:30:48.018870291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:30:53.019121749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:58.000264937Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:30:58.019054636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:30:58.019078199Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:30:58.019102541Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:31:02.563853737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:31:02.574500214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99AB160CB0E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:31:03.019030382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:31:08.019709942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:09.828629605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:31:09.833906197Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99ACC6BE9C30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:31:09.833923426Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:31:09.833941744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:31:09.838332385Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99ACC7036373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:31:13.019203575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:31:18.01913268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:21.759876071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:31:21.767356291Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99AF8E0551DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:31:23.019755592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:31:28.019695696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:28.019694107Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:31:28.019724797Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:31:33.019799652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:31:38.01927583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:42.935603353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:31:42.94319762Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99B47C34912E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:31:43.019891037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:45.377257287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:31:45.382768889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99B50D9CA0F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:31:48.019854735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:31:53.019217959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:58.000221906Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:31:58.018953232Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:31:58.018975455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:31:58.018999494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:31:58.080895209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:31:58.092093404Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99B80325573C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:32:03.019033175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:08.019766709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:13.019828032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:18.01939598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:23.019672377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:28.019349543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:32:28.019381477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:32:28.019410901Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:32:33.01922392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:32:36.982651096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:32:36.991177899Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99C111B6251A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:32:37.54964277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:32:37.55455023Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99C1334D1553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:32:38.018846931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:43.019883241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:32:48.019055054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:32:48.511403511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:32:48.523134703Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99C3C11222A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:32:53.01898383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:32:58.000266959Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:32:58.018780493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:32:58.018804991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:32:58.018824952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:03.019386907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:08.018881865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:13.01985053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:18.019458086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:23.019107652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:33:23.040253086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:33:23.046403489Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99CBCAD02D91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:33:23.218588566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:33:23.225126929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99CBD579B02E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:33:28.018825284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:33:28.018855471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:33:28.018865754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:33:28.026442312Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-07-02T22:33:28.027908719Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008652887s
level=info ts=2026-07-02T22:33:29.076038546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:33:29.085246197Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99CD32C311C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:33:33.019386353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:38.019243234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:43.019693237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:48.019324467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:33:53.019808105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:33:58.000241222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:33:58.018892035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:33:58.018920686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:33:58.018918608Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T22:33:58.018942188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:33:58.019038617Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-02T22:33:58.01904684Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:33:58.019063553Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.32µs success=true
level=warn ts=2026-07-02T22:34:03.019753322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:08.019421472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:13.019378337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:15.800938431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:34:15.807917189Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99D813A5529F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:34:18.01885343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:21.426827871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:34:21.43309885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99D962F0AF99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:34:21.433117375Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:34:21.433137996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:34:21.438991901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99D9634CBD10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:34:23.01897041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:25.963529608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:34:25.973054502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99DA718B1B56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:34:28.019341391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:34:28.019368674Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:34:28.019387396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:33.019194645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:38.019031375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:39.71017718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:34:39.718291369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99DDA4D09132, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:34:43.019402045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:48.019264287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:34:53.019118484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:58.001015453Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:34:58.019493204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:34:58.019495711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:34:58.019516724Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:34:59.715191676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:34:59.720053535Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99E24D053A84, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:34:59.821616292Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE99E25315E0B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:35:03.018981756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:07.664052528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="3.6 MB" avg_comp="3.6 MB" total_uncomp="31 MB" avg_uncomp="31 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:35:07.673529089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99E427151AAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:35:08.019259147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:09.635001269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:35:09.641407908Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99E49C5EA5D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:35:13.019848893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:18.019443803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:23.019789969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:28.018964433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:28.018990568Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:35:28.019007816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:35:33.019747086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:38.018852537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:43.019100123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:35:48.018965211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:48.080094511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:35:48.08684878Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99ED8FE7777B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:35:53.019571645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:53.786133292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:35:53.792684447Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99EEE400764C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:35:55.630587917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:35:55.640751352Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99EF52256EB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:35:58.000637215Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:35:58.019013327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:35:58.019033933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:35:58.019048686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:36:03.019478906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:08.019629283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:13.019324098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:18.019795808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:23.019130226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:36:28.018988938Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:36:28.019031161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:36:28.019053196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:36:29.458930392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:36:29.465866293Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99F73247EB81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:36:33.01933988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:38.019383891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:36:38.728191356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:36:38.737308193Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99F95AE76E98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:36:38.737327615Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:36:38.737348334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:36:38.745790336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99F95B6B5F99, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:36:42.299712094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:36:42.305735814Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99FA2F9A68CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:36:43.019218548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:36:48.019421106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:36:49.770270035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:36:49.779737534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE99FBED16DCCA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:36:53.019588093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:36:58.000571575Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:36:58.018913727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:36:58.018948739Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:36:58.018948389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:03.019866766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:08.019259615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:37:10.595395385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:37:10.60472836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A00C65BEDC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:37:12.980187455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:37:12.985122318Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A01543DFA60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:37:12.985138759Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:37:12.985176077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:37:12.989542814Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A0154836EFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:37:13.019376201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:18.019327599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:23.019283551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:37:28.01895869Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:37:28.01896788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:37:28.018988475Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:37:30.906711644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:37:30.914706173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A0580EC5394, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:37:32.311381351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:37:32.316874895Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A05D48052A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:37:33.019452764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:38.019581249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:43.019626611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:48.018968292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:37:53.019121914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:37:53.346047003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:37:53.357983784Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A0ABAA50114, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:37:58.000519295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:37:58.018963394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:37:58.018985034Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:37:58.019009181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:38:03.019628605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:07.822930195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:38:07.828755012Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A0E192D45F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:38:08.019579067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:38:13.019308997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:15.191096732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:38:15.197233998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A0FD05ED9C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:38:18.019811573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:38:23.019749832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:28.019453057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:38:28.01949231Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:38:28.019516043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:28.028692082Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-07-02T22:38:28.030348318Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011142597s
level=warn ts=2026-07-02T22:38:33.019294196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:38:35.50645321Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9A148AE411EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T22:38:37.335810284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:38:38.019093055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:38:39.211020631Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A1567B50874, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:38:43.019653581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:38:48.019455456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:38:53.019735921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:58.001120368Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:38:58.019521408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:38:58.019529825Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T22:38:58.019544557Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:38:58.019598197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:38:58.019625673Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-02T22:38:58.01964636Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:38:58.019663786Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.241µs success=true
level=info ts=2026-07-02T22:38:58.826941922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:38:58.832713219Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A19F94086EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:39:03.019746023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:39:08.019118732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:11.039841624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:39:11.048679548Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A1CD160EE0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:39:13.019703042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:39:18.019832975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:20.013321099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:39:20.563097219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A1F08797F48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:39:23.019024067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:28.019511508Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:39:28.0195452Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:39:28.019563041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:39:33.01982655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:39:38.019311433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:39:43.019780064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:46.234649116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="25 kB" avg_comp="25 kB" total_uncomp="333 kB" avg_uncomp="333 kB" idle=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:39:46.242057671Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2503110679, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:39:46.534679284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="193 B" avg_comp="193 B" total_uncomp="274 B" avg_uncomp="274 B" idle=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:39:46.536957525Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2514A59066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:39:46.835717235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="229 B" avg_comp="229 B" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" idle=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:39:46.838001388Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2526977B2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:39:47.134805646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:39:47.137123618Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A25386B769A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:39:47.434888647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="132 B" avg_comp="132 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:39:47.437195152Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A254A4DAA19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:39:47.734827071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:39:47.737028978Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A255C2E2E5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:39:48.019027695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:48.035179939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:39:48.037352457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A256E14AD02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:39:48.335394179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:39:48.337653608Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A257FF99EEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:39:48.635651712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:39:48.637828762Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2591DEA46D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:39:48.935182807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="138 kB" avg_uncomp="138 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:39:48.937530696Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A25A3BC1931, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:39:49.23554648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="109 B" avg_comp="109 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:39:49.238025654Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A25B5A3C380, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:39:49.534840784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:39:49.537062597Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A25C777DD1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:39:49.835028277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:39:49.837206502Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A25D95B0F0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:39:51.635083789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:39:51.639067534Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2644C0019B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:39:53.019847989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:54.482504395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:39:54.48796701Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A26EE8E1DDE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:39:57.255520087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:39:57.260515983Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2793CF2B52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:39:58.00059786Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:39:58.018946956Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:39:58.018972688Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:39:58.018988078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:39:59.805822117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:39:59.808278319Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A282BAB1F2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:40:00.567262586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:40:00.569703833Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A28590E9768, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:40:00.719249115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:40:00.721567743Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A28621BF80D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:40:01.606073194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:40:01.608535018Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2896F9CF69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:40:02.221022895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:40:02.223712259Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A28BBA3159B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:40:02.880367582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:40:03.019861474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:03.123106665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:40:03.12791255Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A28F18B41CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:40:03.427193847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:40:03.429478521Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A290384B657, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:40:03.454511537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:40:03.456441118Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A290521F4BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:40:03.534632536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:40:03.53647611Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2909E76754, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:40:03.713395527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:40:03.715519411Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A29149204AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:40:03.986941416Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2924BF5B4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:40:05.144258808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:40:05.148744169Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2969FEBA75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:40:05.332247526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:40:05.334388874Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A29750E4CB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:40:08.019729546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:12.086674181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:40:12.088981762Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2B07A8C7E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:40:13.019092385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:40:18.019595537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:40:23.018968415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:23.974773108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:40:23.977677145Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2DCC48CF64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:40:25.662551366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:40:25.664769103Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2E30D813FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:40:27.129888388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:40:27.132146563Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2E884DB840, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:40:28.019383457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:40:28.019422233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:40:28.019419437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:30.115414881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:40:30.117712321Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2F3A430527, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:40:31.757030825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:40:31.759271226Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2F9C1B2F5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:40:32.214847777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:40:32.217090644Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A2FB764AB1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:40:33.018981084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:33.582787445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:40:33.585188936Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3008EF0E1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:40:34.105402926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:40:34.107613363Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3028147636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:40:36.671357116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:40:36.673628315Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A30C106648A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:40:37.425536701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:40:37.427717469Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A30EDF9218B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:40:37.578936752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:40:37.580841065Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A30F71AB469, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:40:38.018981906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:39.097860747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:40:39.100140008Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3151A75385, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:40:39.518195515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:40:39.520668631Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A316AB619D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:40:43.018896583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:43.070499837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:40:43.072663729Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A323E707A20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:40:48.019780266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:48.74632322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:40:48.750179893Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3390D70DC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:40:48.750199064Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:40:48.750219549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:40:48.75700093Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A339140CB40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:40:53.019828899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:40:53.514170929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:40:53.519547912Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A34AD1F2FB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:40:56.472898471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:40:56.47813862Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A355D74F0E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:40:58.000610635Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:40:58.018984738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:40:58.019018111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:40:58.019026479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:00.455472608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:41:00.462318002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A364AEFA9E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:41:03.019670766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:06.762922225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:41:06.770175245Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A37C2E95817, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:41:07.662555939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:41:07.666964268Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A37F85EC547, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:41:08.018855422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:41:13.019319301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:41:18.018811864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:20.698330598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:41:20.701378484Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3B01474109, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:41:20.882798829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:41:20.884844715Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3B0C3892E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:41:20.943046148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:41:20.944963464Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3B0FCE8821, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:41:22.134405387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:41:22.136480433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3B56D2F8AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:41:23.019127436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:23.072339404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:41:23.074465929Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3B8EBA9EF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:41:28.019583764Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:41:28.019594333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:28.019615429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:41:29.448987034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:41:29.45122693Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3D0AD03DBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:41:31.057470489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:41:31.059559953Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3D6AAE9DA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:41:31.236065816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:41:31.238289185Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3D755492BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:41:32.1221869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:41:32.124630468Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3DAA275C7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:41:33.019325296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:33.976755783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:41:33.978903939Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3E18AFE1E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:41:34.119585232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:41:34.121651934Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3E21328237, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:41:37.478577983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:41:37.480876998Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3EE96B32F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:41:38.0193341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:39.253553702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:41:39.255789507Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3F53353414, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:41:39.280827638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:41:39.283244483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A3F54DA0684, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:41:43.019609488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:45.273134942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:41:45.283843348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A40BA80D523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:41:45.301984705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:41:45.307744398Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A40BBEF739B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:41:48.019685284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:41:53.019690739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:41:58.000223024Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:41:58.019737256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:41:58.019760863Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:41:58.019765829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:02.525038558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:42:02.531029985Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A44BE8542BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:42:03.019037642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:06.259907749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:42:06.262022212Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A459CE8C7A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:42:08.019474049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:10.752167576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:42:10.754417682Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A46A8ACDA94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:42:11.475549563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:42:11.477769917Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A46D3CB4890, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:42:13.019134914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:14.622927775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:42:14.625112156Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A478F632A78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:42:16.536394568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:42:16.53850422Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A48016FCC36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:42:17.354766545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:42:17.357021812Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4832389BC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:42:18.019809562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:18.088936368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:42:18.091180228Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A485DFAB8E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:42:20.096185969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:42:20.098459037Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A48D59F3652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:42:22.87791291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:42:22.880112584Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A497B6B8004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:42:23.019688443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:25.215747104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:42:25.218091437Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4A06C61A63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:42:25.79028294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:42:25.792433708Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4A2902EB29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:42:26.671741527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:42:26.674002123Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4A5D8DFD16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:42:28.01898487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:28.019001167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:42:28.019032984Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:42:33.019898252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:33.103169398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:42:33.106025084Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4BDCEE1343, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:42:35.361253771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:42:35.368658622Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4C63CC845F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:42:36.228064211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:42:36.232665146Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4C974C1B8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:42:38.019344043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:43.019087125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:42:43.019096045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:42:43.026741438Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4E2C4114F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:42:43.026757796Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:42:43.026779442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:42:43.036306568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A4E2CD37DCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:42:48.018944427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:42:53.019066018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:53.688505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:42:53.695229064Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A50A823ED95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:42:57.24174262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:42:57.243881504Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A517BAA394A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:42:57.696698269Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9A5196A7CD5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T22:42:58.001177756Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:42:58.019617567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:42:58.019620928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:42:58.01965718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:42:58.765900869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:42:58.768119562Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A51D68359C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:42:59.31172524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:42:59.313884166Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A51F70BA31B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:43:01.568919627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:43:01.689185317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:43:01.694059937Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5284E7EC57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:43:02.287060381Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A52A8407ED1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:43:02.354323134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:43:02.35889814Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A52AC8AE0B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:43:03.019368557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:43:08.019776119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:10.055996919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:43:10.058338026Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A547776133C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:43:11.786992818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:43:11.789387513Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A54DEA32F93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:43:11.952918703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:43:11.955109634Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A54E884FC21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:43:13.018982934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:14.943578985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:43:14.945886399Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A559AC8E934, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:43:17.799903954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:43:17.802248236Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5645085E7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:43:18.019290022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:18.365714457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:43:18.367973158Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5666C0C301, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:43:22.763768993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:43:22.766075159Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A576CE68B10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:43:23.019173117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:26.062265876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:43:26.064666591Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A583182D4F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:43:28.01920266Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:43:28.019232568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:43:28.01922616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:28.028214139Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-07-02T22:43:28.029799911Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010010973s
level=info ts=2026-07-02T22:43:30.158903534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:43:30.165831036Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5925F67129, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:43:31.364173149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:43:31.372131011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A596DDA4955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:43:32.113245722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:43:32.118255422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A599A5608BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:43:33.019204702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:43:38.019105584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:43:43.019824374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:45.154338208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:43:45.159878003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5CA3ACE854, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:43:46.865834915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:43:46.868237341Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5D0980BD24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:43:48.018874557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:49.983872308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:43:49.986106337Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5DC358C2EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:43:50.92713468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:43:50.929502774Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5DFB934E92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:43:51.739636525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:43:51.742010471Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5E2C00BCE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:43:53.019040256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:54.676275199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:43:54.678682884Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A5EDB0B3DD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:43:58.000362873Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:43:58.018812994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:43:58.018836867Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:43:58.018839231Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T22:43:58.018851789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:43:58.01895509Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-02T22:43:58.018965675Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:43:58.018984047Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.59µs success=true
level=info ts=2026-07-02T22:44:01.477439498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:44:01.479870508Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A60706C0B24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T22:44:03.019190039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:03.903398363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:44:03.905527632Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6101030F97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:44:05.806769246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:44:05.809123154Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A61727783DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:44:06.754640723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:44:06.75702242Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A61AAF642F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:44:08.018855046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:08.791418129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:44:08.793786801Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A62245D6223, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:44:10.905755898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:44:10.908048966Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A62A263BEEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:44:12.769203191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:44:12.77150428Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6311750CA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:44:13.019091524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:17.422013286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:44:17.424404944Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6426CB5BD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:44:18.019625539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:21.143047484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:44:21.147328292Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6504AED4B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:44:23.019133935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:25.318516273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:44:25.329121016Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A65FDF1A9D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:44:28.019033082Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:44:28.019062589Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:44:28.019079421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:29.287299988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:44:29.292588665Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A66EA2E2105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:44:29.292606259Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:44:29.292626715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:44:29.29702407Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A66EA75F8D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:44:33.019793301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:33.168505898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:44:33.172574301Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A67D1754240, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:44:38.018876255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:38.656570775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:44:38.658763591Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6918753205, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:44:38.980542165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:44:38.982647954Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A692BC4194E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:44:41.06389639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:44:41.066488678Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A69A7F703AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:44:42.225787267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:44:42.227924574Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A69ED332825, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:44:43.019411563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:44.606271504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:44:44.608916933Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6A7B1D9CEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:44:47.056696206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:44:47.062419064Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6B0D59ABD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:44:48.01955981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:44:53.019112079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:53.257658898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:44:53.259747971Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6C7EBF12EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:44:53.889664146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:44:53.891894334Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6CA46C1DFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:44:55.137198711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:44:55.13963875Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6CEEC9E2E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:44:55.397524585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:44:55.399814929Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6CFE4C0E2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:44:58.000951815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:44:58.019390059Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:44:58.019412756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:44:58.019447637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:44:58.651528025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:44:58.653864897Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6DC041A8C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:45:03.01900657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:03.52165401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:45:03.523949764Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6EE288F987, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:45:06.962361772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:45:06.966711743Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6FAFBB9624, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:45:07.484104776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:45:07.489566053Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6FCEE7D256, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:45:08.019774905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:08.075930953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:45:08.078621206Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A6FF203A48F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:45:09.267499288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:45:09.27179496Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7039247D16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:45:13.019852227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:15.019995895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:45:15.02669877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7190281A0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:45:16.491491416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:45:16.495918445Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A71E7BA8D18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:45:18.019593647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:45:23.019053974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:28.01953549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:45:28.019585947Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:45:28.019675724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:28.40219072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:45:28.405080791Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A74AD90AAB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:45:33.01911112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:35.92366688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:45:35.925984114Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A766DD96336, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:45:36.012456856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:45:36.014526096Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A76732157EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:45:36.733250632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:45:36.735766998Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A769E1C1F9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:45:37.841798261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:45:37.844031296Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A76E02CDBF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:45:38.019078945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:39.863520214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:45:39.865818903Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7758AF8B40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:45:39.962012076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:45:39.964067287Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A775E8AAC61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:45:43.019617344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:44.687115585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:45:44.689497459Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A78783277D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:45:45.455551134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:45:45.457698874Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A78A5FD8DE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:45:46.758342027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:45:46.760512505Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A78F3A449C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:45:48.019292133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:51.377447834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:45:51.379785225Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7A06F8A860, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:45:53.019646145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:45:58.000237066Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:45:58.019726102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:45:58.019745853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:45:58.01976638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:00.509749021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:46:00.511903803Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7C274A8A79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:46:02.023083293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:46:02.025356725Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7C817E35F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:46:02.453506411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:46:02.45624177Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7C9B2BFD05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:46:03.019330748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:03.749018687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:46:03.756347717Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7CE8A9B48B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:46:04.841334327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:46:04.850177588Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7D29DCEB9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:46:08.019482195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:46:13.019832811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:13.051992331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:46:13.056535975Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7F1300903E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-02T22:46:13.056550971Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:13.056573782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:46:13.058438669Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A7F1320D0D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:46:18.019087881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:19.05812933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:46:19.060385514Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8078DDC062, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:46:19.060401844Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:19.060420971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:46:19.062245263Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8078FB62F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:46:22.462099391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:46:22.464461441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8143C3270F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-02T22:46:22.46447816Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:22.464496754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:46:22.466374569Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8143E218DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:46:23.019324148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:25.192197066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:46:25.19461248Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A81E67E0546, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:46:25.194628883Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:25.19464705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:46:25.196558435Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A81E69DC94A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:46:25.811079718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:46:25.81328718Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A820B5EB70B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:46:25.813301904Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:25.813319692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:46:25.815143815Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A820B7C5589, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:46:25.854299693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:46:25.856207885Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A820DEDF015, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:46:26.004842898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:46:26.006812658Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8216E92352, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:46:26.006824578Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:26.114014958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:46:26.116013098Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A821D6B1594, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:46:28.018822999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:46:28.018853096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:46:28.018840906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:29.259426028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:46:29.261758203Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A82D8E97909, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:46:29.26177405Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:29.261791783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:46:29.263607615Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A82D908362C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:46:30.168050108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:46:30.170458569Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A830F128007, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:46:30.170475486Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:30.170492892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:46:30.172600249Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A830F35186C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:46:30.367991377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:46:30.370240295Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A831AFBA6C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:46:31.879832323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:46:31.882310431Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A83751AA119, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:46:31.882326562Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:31.882345354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:46:31.884443376Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A83753CF1FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:46:32.95561933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:46:32.957883121Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A83B538C475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:46:32.957899427Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:32.957917182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:46:32.959734203Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A83B556C44C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:46:33.019787801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:33.224066495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:46:33.22627461Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A83C536E04B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:46:35.554738781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:46:35.557209691Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A845024AD4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:46:35.557227117Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:35.557245635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:46:35.559079937Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8450456665, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:46:38.019190578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:39.382279706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:46:39.384597075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A853447BF88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:46:39.384615895Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:39.384633587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:46:39.386450118Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8534665AD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:46:40.664176111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:46:40.666536643Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8580B0BEA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:46:41.833908036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:46:41.836296849Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A85C668F47D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:46:42.795889758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:46:42.798345155Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A85FFC10B4E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:46:43.019042104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:43.143306667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:46:43.14565566Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A861474C742, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:46:43.494649884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:46:43.49697971Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8629658E46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:46:43.704425271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:46:43.706628843Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8635E56213, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:46:43.706642424Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:43.706659164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:46:43.708462588Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A863603A4BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:46:46.881429463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:46:46.883737954Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A86F3439ABB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:46:48.019487116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:49.311935161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:46:49.314701425Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8784293A8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:46:50.004675833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:46:50.010296214Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A87AD9DD1F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:46:50.052313251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:46:50.054388805Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A87B04152D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:46:50.562627964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:46:50.564853756Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A87CEAD6945, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:46:53.019438758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:55.78717718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:46:55.789550918Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8906182399, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:46:58.000609098Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:46:58.018957378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:46:58.018982605Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:46:58.018996687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:46:58.201658725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:46:58.20390382Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8995FFBEB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:46:58.203918244Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:46:58.203935626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:46:58.205934027Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A89961FDF79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:46:59.388142775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:46:59.395041587Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A89DCFEDD2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:47:01.399543247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:47:01.406331535Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8A54DDC7B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:47:03.019805808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:03.566265164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:47:03.571204815Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8AD5E92B4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:47:03.796630779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:47:03.798952032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8AE37CA950, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:47:06.473573335Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9A8B82EB0FE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T22:47:08.019411503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:08.617536728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:47:08.619803049Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8C02D53BF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:47:09.011984126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:47:09.01437007Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8C1A588910, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:47:09.35910618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:47:09.361561476Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8C2F0A3241, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:47:10.922799362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:47:10.925246751Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8C8C3E42B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:47:11.443864264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:47:11.446341909Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8CAB4CC87F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:47:11.53352156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:47:11.535708567Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8CB0A291BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:47:13.01921667Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:15.908027009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:47:15.910856787Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8DB565D534, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:47:16.086507188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:47:16.089063403Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8DC006CBA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:47:16.947666823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:47:16.95008434Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8DF359F3BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:47:18.018805728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:47:23.019703957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:25.566291347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:47:25.568789737Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A8FF51063F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:47:28.019471471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:28.019496079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:47:28.019514819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:47:28.362110995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:47:28.364773674Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A909BB731BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:47:32.416388767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:47:32.418754228Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A918D5BDA23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:47:33.019194327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:33.993333138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:47:33.996177872Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A91EB607835, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:47:38.019558194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:42.961428406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:47:42.973910079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A94027D6EA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:47:43.018939197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:45.371783318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:47:45.377819855Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9491C9869B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:47:45.377837932Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:47:45.377859512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:47:45.383323488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A94921ECADC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:47:48.01973313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:49.002516759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:47:49.007177211Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A956A1B1329, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:47:49.905518767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:47:49.907745772Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A959FC9BB67, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:47:50.836287031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:47:50.838648951Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A95D745EC10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:47:53.019311989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:53.926126569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:47:53.928362612Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A968F6F8732, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:47:58.00048399Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:47:58.018949104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:47:58.018966772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:47:58.01898715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:48:01.381260467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:48:01.385573047Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A984BEA19AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:48:03.019811745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:03.279912798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:48:03.284892989Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A98BD1ECE51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:48:06.185765805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:48:06.18814264Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A996A2AE997, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:48:06.851186526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:48:06.853347039Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9991D222F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:48:07.494187279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:48:07.496454183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A99B82652FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:48:07.980691652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:48:07.982890749Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A99D52573DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:48:08.019602624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:08.052746378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:48:08.054775857Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A99D96FE09D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:48:08.48265329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:48:08.484724802Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A99F30F5A06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:48:12.326953273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:48:12.329406752Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9AD836AAE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:48:13.018970613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:17.009057449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:48:17.011382344Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9BEF491A96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:48:18.019458023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:22.383508165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:48:22.385943616Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9D2FA27D8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:48:23.019697702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:24.667567022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:48:24.671741162Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9DB7E07C12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:48:24.740899165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:48:24.747239438Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9DBC62B82E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:48:28.0189618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:28.018971209Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:48:28.01899741Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:48:28.028466261Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-07-02T22:48:28.030110568Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010968805s
level=info ts=2026-07-02T22:48:32.34970658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:48:32.359257688Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9A9F8211FCF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:48:33.019191901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:35.134884359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:48:35.137223422Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA027ACCBF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:48:35.217527679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:48:35.225351723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA02CEDDFD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:48:35.705596041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:48:35.710050351Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA049D1042F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:48:38.013037138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:48:38.015262844Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA0D3375E77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:48:38.019425322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:48:43.019864742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:48:48.019266396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:49.24705904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:48:49.250265075Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA370DFAE20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:48:53.0196682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:56.308102482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:48:56.310441275Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA515B17A37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:48:57.214964575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:48:57.217246689Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA54BBE79FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:48:58.00034513Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:48:58.01875564Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T22:48:58.018787095Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:48:58.018809593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:48:58.018821506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:48:58.018858906Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-02T22:48:58.018866646Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:48:58.018884598Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.016µs success=true
level=info ts=2026-07-02T22:49:00.415557753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:49:00.417861971Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA60A847406, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:49:00.626338117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:49:00.628675966Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA61714DAFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:49:02.194593275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:49:02.196864981Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA6748D02B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:49:02.249907642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:49:02.251854441Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA677D70318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:49:02.749207834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:49:02.751403148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA6959C3D82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:49:03.019693993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:03.377532041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:49:03.37980266Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA6BB0F5B63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:49:08.019003195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:49:13.019316838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:17.134605238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:49:17.137573682Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AA9EF17A600, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:49:18.019463104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:18.151781981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:49:18.155622767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAA2BC40919, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:49:19.096025767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:49:19.101296422Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAA6421A35E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:49:19.515475611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:49:19.524144946Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAA7D574441, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:49:19.524177953Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:49:19.524201876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:49:19.532869895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAA7DDC3263, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:49:19.65019002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:49:19.654677419Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAA851D4C28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:49:23.019076271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:26.351765017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:49:26.354321457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAC147014B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:49:28.018811992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:49:28.018842477Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:49:28.018839415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:32.306793486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:49:32.309267871Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAD7761D9CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T22:49:33.018994107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:34.728736246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:49:34.736037947Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAE08076211, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:49:37.299638988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:49:37.305395439Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAEA130380F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:49:38.019643568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:40.776418498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:49:40.778683622Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AAF7036B189, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:49:43.018981724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:43.433828887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:49:43.436296279Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB00E9BE1EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:49:46.180764077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:49:46.183092255Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB0B256FD3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:49:46.673414264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:49:46.675766799Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB0CFB3F008, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:49:48.019710415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:49.603124804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:49:49.605609865Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB17E564BC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:49:50.57407046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:49:50.576446016Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB1B833AA32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:49:51.526062099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:49:51.528346007Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB1F0F0676A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:49:51.698997343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:49:51.70112201Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB1FB3D603D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:49:53.019642919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:57.901125464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:49:57.903399729Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB36CECF4D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:49:58.000881079Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T22:49:58.019287304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:49:58.019298571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:49:58.019321331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:50:03.020655805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:04.116407483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:50:04.118674912Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB4DF625743, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:50:07.093719085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:50:07.101639327Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB5912E818F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:50:08.018889603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:11.40051525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:50:11.405766732Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB691B9646A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:50:12.450539294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:50:12.456484996Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB6D05AD5CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:50:13.019510168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:14.719588605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:50:14.724054577Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB7578282B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:50:18.019327181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:19.430734075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:50:19.432940844Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB8702F2601, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:50:20.119898302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:50:20.122136841Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AB8994272AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:50:23.018992413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:28.019288739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:50:28.019319731Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:50:28.019312429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:30.93226843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:50:30.935949551Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABB1DD0259D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:50:31.690003408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:50:31.692346209Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABB4AE556F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:50:32.281982461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:50:32.284274307Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABB6E2E0DF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:50:33.019131957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:34.216553528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:50:34.218829242Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABBE17DE137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:50:36.810855151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:50:36.813191385Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABC7C1F7A6D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:50:38.019572776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:38.932495535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:50:38.934758943Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABCFA94F183, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:50:39.519412875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:50:39.521669067Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABD1D910D9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:50:43.019496771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:44.331681377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:50:44.333943966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABE3C669D6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:50:45.628749675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:50:45.63132978Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABE89B75DE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:50:48.019529612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:50:49.917103143Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9ABF89315164, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T22:50:51.344810062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:50:51.347098072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ABFDE6A5019, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:50:53.019368017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:53.604604477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:50:53.608321201Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC065315306, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:50:58.00073807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:50:58.019033675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:50:58.019058617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:50:58.019071902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:50:58.401434302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:50:58.406029424Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC18327EE3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:50:58.583181303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:50:58.58587962Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC18DE041A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:50:58.585894601Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:50:58.58591365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:50:58.590305897Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC18E259DF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:51:03.019272805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:04.500264474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:51:04.50479682Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC2EEAD512A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:51:05.522563708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:51:05.524858258Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC32B7A85ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:51:06.3850801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:51:06.391844118Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC35F274481, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:51:08.018995522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:11.630458141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:51:11.635832338Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC497B8468C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:51:13.019047146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:17.503667306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:51:17.508507609Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC5F5C1CA72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:51:18.019119891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:19.482003457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:51:19.484243049Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC66B84EFC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-07-02T22:51:23.019761568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:23.442563683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:51:23.444784716Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC757957137, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-07-02T22:51:23.570005261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:51:23.571990585Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC75F2BEE20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:51:24.82599413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:51:24.828275837Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC7AA0B9C1E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:51:26.780852486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:51:26.783056745Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC81E904BD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:51:26.853426757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:51:26.855280462Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC822E03E79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:51:28.019585126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:51:28.019614082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:51:28.019607014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:31.071544898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:51:31.073783055Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC91E5004DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:51:33.019349591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:33.581797064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:51:33.583909717Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AC9B3EDD1A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:51:36.029992307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:51:36.032252131Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACA45DBC5EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:51:38.019410824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:42.410510753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:51:42.41268636Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACBC22A6560, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:51:43.019262673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:46.018822166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:51:46.021048126Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACC993CED1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:51:48.019329229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:48.050516655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:51:48.053087622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACD125AEB76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:51:48.494139239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:51:48.502528407Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACD2D23D374, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:51:51.678136016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:51:51.682702482Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACDEAB22FA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:51:53.019059535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:53.091257306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:51:53.097980571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACE3F0D426B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:51:54.521804414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="86 kB" avg_uncomp="86 kB" idle=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:51:54.526686603Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACE943465F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:51:54.822540194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" idle=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:51:54.824938293Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACEA5FC801E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:51:55.122749768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="126 B" avg_comp="126 B" total_uncomp="10 kB" avg_uncomp="10 kB" idle=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:51:55.125031508Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACEB7E0035F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:51:55.42185761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="89 kB" avg_comp="89 kB" total_uncomp="421 kB" avg_uncomp="421 kB" idle=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:51:55.424373911Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACEC9B66812, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:51:58.000397278Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:51:58.019775858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:51:58.019808448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:51:58.019808806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:51:58.247826754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:51:58.249997189Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACF7223ECA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:51:59.742589694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:51:59.744887361Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ACFCB3D2C21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-07-02T22:52:02.383888394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:52:02.386348664Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD068AD221D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:52:03.019251898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:05.198028533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:52:05.200390525Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD1106A199B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:52:08.019413586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:09.402075255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:52:09.404203585Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD20AFCBF62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:52:09.632748957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:52:09.635004449Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD218BC7E12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:52:10.3850067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:52:10.387567319Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD24595F84E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T22:52:13.01961609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:13.714597264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:52:13.716812124Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD30C083E63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:52:14.065791355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:52:14.067931898Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD320F61039, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:52:14.331569121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:52:14.33385136Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD330CF4545, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:52:14.832144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:52:14.834351755Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD34EA44637, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T22:52:18.018835859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:18.88743233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:52:18.88957042Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD4405B4CD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:52:18.982631266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:52:18.984655413Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD446067475, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:52:19.811189907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:52:19.813329739Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD4776A9080, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:52:22.443686538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:52:22.445933289Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD51453FD58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-07-02T22:52:23.019751074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:28.019255242Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:52:28.01928512Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:52:28.019282437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:28.437423934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:52:28.440032421Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD6799A922B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:52:33.019122706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:34.373254553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:52:34.378038037Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD7DB87A0D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:52:34.379000297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:52:34.381432167Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD7DBB7D99E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:52:35.782555785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:52:35.785178622Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD82F64C9E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:52:36.938412979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:52:36.94066127Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD874465017, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:52:36.94067463Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:52:36.940691552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:52:36.942489146Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AD87464FBD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:52:38.019407626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:52:43.019755527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:44.86942488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:52:44.87179761Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADA4D0153AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:52:44.871814047Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:52:44.871831411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:52:44.873777962Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADA4D22DA8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:52:45.860571798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:52:45.864569196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADA882E1F1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:52:46.327029467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:52:46.405565891Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADAA8707F2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:52:46.657174869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:52:46.661427135Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADAB7B0207D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T22:52:48.019595684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:48.943620884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-07-02T22:52:48.945762834Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB3FD846D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-07-02T22:52:48.945778895Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:52:48.945798589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:52:48.947588158Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB3FF5F01B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:52:49.480192715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:52:49.482292821Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB5FD2828C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:52:49.482308733Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:52:49.482326185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:52:49.484090293Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB5FF02F04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:52:50.333192119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:52:50.335435285Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB92AC8018, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:52:50.375459065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:52:50.37757631Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADB95308EA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:52:51.355256384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:52:51.357575525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADBCF97E0DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=warn ts=2026-07-02T22:52:53.019845085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:54.851991476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:52:54.854381168Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADCA004FEED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:52:54.854398641Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:52:54.854417873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:52:54.856184385Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADCA023AA4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:52:56.516714203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:52:56.518902339Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADD033D1B01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:52:58.00025562Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:52:58.019638017Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:52:58.019651247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:52:58.019670491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:53:01.573924699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:53:01.57620357Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADE30AC7FD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:53:01.576223398Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:01.576242378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:53:01.578015032Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADE30CA1DD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:53:03.01959829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:04.946734468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:53:04.949006675Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADEF9B519BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:53:04.949021848Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:04.949040893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:53:04.950788196Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADEF9D38CDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:53:05.764161585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:53:05.766322744Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADF2A6CC856, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:53:05.766342253Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:05.766360828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:53:05.768067973Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADF2A89D375, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:53:05.853389923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:53:05.855268403Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADF2FBBE093, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:53:06.737935354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:53:06.739971946Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADF6477287B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:53:07.839683201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:53:07.841718121Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADFA62222BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:53:08.019539882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:08.537970246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:53:08.540245087Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9ADFCFC2B402, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:53:11.053333858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-07-02T22:53:11.055622045Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE065B0236F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-07-02T22:53:11.055635679Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:11.055664875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="324 B" avg_comp="324 B" total_uncomp="102 kB" avg_uncomp="102 kB" idle=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:53:11.057607075Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE065D18124, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:53:11.738312366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:53:11.740544087Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE08E848234, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:53:13.019730142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:13.594184745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:53:13.596369257Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE0FD21AA2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:53:13.596387034Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:13.596408211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:53:13.598173253Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE0FD3EECAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:53:14.116193056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:53:14.118802149Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE11C459802, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:53:14.808658854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:53:14.813667854Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE145AFF57B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:53:14.813687876Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:14.813706044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:53:14.81558865Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE145D03BF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:53:15.038314182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:53:15.040386062Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE153350DFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:53:15.040398934Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:15.114582291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:53:15.116594676Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE157C0319B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:53:16.438431553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:53:16.440650281Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE1A6AB08BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:53:16.440667511Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:16.440684459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:53:16.442501114Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE1A6C867DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:53:16.55665952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:53:16.558789495Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE1ADB60E8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:53:18.019207282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:18.927627314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:53:18.929965659Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE23B0A903E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:53:21.007476817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:53:21.009823968Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE2B7024187, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:53:23.019502748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:25.784053415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:53:25.786488848Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE3D3B78573, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:53:28.018994025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:53:28.019035091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:53:28.019039684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:28.025962705Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-07-02T22:53:28.027651815Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008642011s
level=info ts=2026-07-02T22:53:28.681259701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:53:28.68359668Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE4806724E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:53:28.686629757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:53:28.688752455Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE480B62D4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:53:30.937908994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:53:30.940266928Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE506E88DC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:53:31.149892242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:53:31.152189115Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE5138B03AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:53:31.505183345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:53:31.507393533Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE528B6562A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:53:31.610731144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:53:31.612729001Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE52EFF8E05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:53:32.542120514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:53:32.544799571Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE56689C267, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T22:53:33.019042248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:33.53948956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:53:33.541725079Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE5A1F825B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:53:35.640836426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:53:35.643310216Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE61F3A6C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T22:53:38.019109913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:38.490499926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:53:38.494373204Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE6C92B3E78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:53:39.495604264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:53:39.501585211Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE70533816E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:53:40.159268115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:53:40.161662006Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE72C8ACDCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:53:40.630086293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:53:40.636722657Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE748DCBDE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:53:43.019328082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:45.986886906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:53:45.991537289Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE888089E02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:53:48.019494239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:49.920941761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:53:49.923276989Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE97261E565, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:53:51.380119255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:53:51.383286574Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE9C95A80D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:53:51.862537002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:53:51.865016448Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE9E61E3780, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:53:51.865032577Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:53:51.865050902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:53:51.866972472Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AE9E63EE063, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:53:52.738863072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:53:52.741215377Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEA1A588401, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:53:53.019077055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:56.727134701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:53:56.729417287Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEB0810C9C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:53:58.000231493Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:53:58.01882903Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:53:58.018850465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:53:58.01885761Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T22:53:58.018886279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:53:58.018964066Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-02T22:53:58.018972934Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T22:53:58.01899254Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.126µs success=true
level=info ts=2026-07-02T22:53:58.391686953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:53:58.393908189Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEB6B46AB0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:53:59.348101614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:53:59.350439054Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEBA44A07B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:54:00.705198774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:54:00.707465584Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEBF52CB704, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:54:02.164267643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:54:02.166439025Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEC4C23F744, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:54:03.019771492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:54:08.018800088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:09.039253762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:54:09.041611515Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEDE5EC601B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:54:09.077646331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:54:09.079704292Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEDE833F620, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:54:10.007267076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:54:10.009579078Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEE1F9F80CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:54:13.018944788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:13.820117528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:54:13.822473237Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEF02E2E0E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:54:13.909665725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:54:13.912241725Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AEF083D5A3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:54:18.01904954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:18.939475993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:54:18.947604012Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF0345A00C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:54:18.947625908Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:54:18.947645578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:54:18.953233583Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF034B43948, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:54:23.019473678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:24.593342668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:54:24.598711383Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF1852C7473, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:54:27.002393216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:54:27.004655211Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF2149B0D08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:54:28.01953958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:54:28.01956932Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:54:28.019560806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:29.07079512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:54:29.077579416Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF29028CF88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:54:29.559726704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:54:29.564226556Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF2AD2B26FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:54:31.316816613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:54:31.319033681Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF315C38AF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:54:32.946528459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:54:32.948708172Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF376E6B838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:54:33.01972593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:34.029130706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:54:34.033189471Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF3B789DEA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:54:34.286660522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:54:34.291757644Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF3C6F27F09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:54:38.018793706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:54:38.018811003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:54:38.021271161Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF4A53D528A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:54:38.212945557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:54:38.215265384Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF4B0CE4706, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:54:39.158846365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:54:39.16129447Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF4E9303E53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:54:41.185986121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:54:41.188508819Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF56206115D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T22:54:43.019187466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:43.260610791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:54:43.26300077Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF5DDAC8061, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:54:45.959739843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:54:45.962206612Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF67E8EBC2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:54:48.019090357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:51.131850539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:54:51.134359181Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF7B2D7EE96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:54:53.019366614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:53.801526254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:54:53.803949574Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF851F72CC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:54:55.858252515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:54:55.860768127Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF8CC8EABD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:54:56.16367526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:54:56.167665821Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF8DEDA299F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:54:56.395308099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:54:56.401294672Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AF8ECC7D7D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:54:58.000879708Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:54:58.019114655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:54:58.019135348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:54:58.019144489Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:55:01.524003242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:55:01.526373273Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFA1E42891D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:55:01.627500569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:55:01.629431601Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFA246957C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:55:02.396141663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:55:02.398933867Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFA52451A17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:55:03.019708482Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:55:03.823190499Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9AFAA72A57B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T22:55:05.023636661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:55:05.028519943Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFAEEFF0D4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T22:55:08.019256055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:55:13.019330638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:13.406189958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:55:13.408506448Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFCE27D9F36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:55:18.019466677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:19.765887506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:55:19.768345662Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFE5D90544C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:55:22.195102141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:55:22.197421805Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFEEE5A5439, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:55:22.787771836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:55:22.790136471Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFF11AD9C18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:55:22.839501387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:55:22.841498875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9AFF14BF3253, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T22:55:23.01984342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:27.668563385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:55:27.676388772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0034DB300D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:55:28.018800124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:55:28.018816082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:28.018826519Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:55:28.434382922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:55:28.440244688Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B00627222A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:55:30.425095689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:55:30.427704767Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B00D8E7CFF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:55:30.725504776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:55:30.727885354Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B00EACDCC34, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:55:31.38866421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:55:31.391139668Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B01125608F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:55:32.163359772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:55:32.165724379Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0140812F70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:55:33.017073964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:55:33.019434501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T22:55:33.019489428Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0173644A75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:55:37.337411391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:55:37.339957512Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0274E8EE4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:55:37.503470183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:55:37.505667939Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B027ECB5102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:55:38.01919459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:40.325289346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:55:40.327764725Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B032700591A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:55:43.019273387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:46.256324986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:55:46.258758452Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0488833219, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T22:55:48.019349307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:49.492337533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:55:49.494856254Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B054966307D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:55:49.966874284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:55:49.970895322Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0565C69666, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:55:53.019843685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:56.976267171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:55:56.982444608Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0707B1685B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:55:58.000410234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:55:58.019795845Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:55:58.019819288Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:55:58.019849013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:55:58.428599542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:55:58.431381014Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B075E101F94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:56:02.61132319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:56:02.616128227Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B08577D685B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:56:02.849634517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:56:02.851815899Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B08658A9E66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:56:03.019173922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:03.556193713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:56:03.558332076Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B088FA7E474, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T22:56:08.01959425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:08.784265901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:56:08.790877132Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B09C78A7604, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:56:08.790894905Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:56:08.790916594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:56:09.024380341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B09D574C6E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:56:10.325716777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:56:10.330473294Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0A234C2E03, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:56:11.492250238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:56:11.494558501Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0A68AF5836, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:56:13.019089787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:15.085058461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:56:15.087298762Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0B3ED52F3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:56:18.018872298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:18.420410585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:56:18.422701321Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0C05A2AA12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:56:18.612241001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:56:18.614315912Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0C110EE19D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:56:19.206092815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:56:19.208678947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0C34797864, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:56:20.723951797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:56:20.726326773Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0C8EF11CBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:56:21.289014193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:56:21.291599385Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0CB0A13551, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:56:21.745657398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:56:21.747993834Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0CCBD6363E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:56:23.019101868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:27.448780898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:56:27.45624879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0E2013BC8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:56:28.01895844Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:56:28.018985121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:56:28.019047551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:28.50326974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:56:28.507592405Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0E5EBFCF92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:56:33.01965392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:34.280194596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:56:34.282393599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0FB6F34B79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:56:35.382294672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:56:35.384501193Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B0FF8A4459A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:56:36.254913456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:56:36.257065563Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B102CA6F9A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:56:38.018806457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:41.476583989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:56:41.478826127Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1163E41B83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:56:43.018972319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:45.162427206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:56:45.164799056Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B123F9579BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:56:46.559515625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:56:46.561680953Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1292D9DA3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:56:47.341500675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:56:47.344326547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B12C17DAD2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:56:48.018868463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:49.802370973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:56:49.809952799Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B135476E0BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:56:53.019015167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:55.902413688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:56:55.907482075Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B14BFE4F2E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:56:58.000617072Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:56:58.018860475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T22:56:58.018856148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:56:58.018883351Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:56:58.306166531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:56:58.308413027Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B154F02C0BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:56:59.407051969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:56:59.409411521Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1590A1CBB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:57:00.552490094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:57:00.554746186Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B15D4E80CC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:57:00.801538925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:57:00.803752799Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B15E3BF371B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:57:01.788194742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:57:01.790742613Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B161E915130, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:57:02.427718162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:57:02.430048961Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1644AE152A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T22:57:03.019793134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:05.395126485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:57:05.39752007Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B16F58DC687, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:57:06.416267732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:57:06.418599385Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B17326A58E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:57:06.692394123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:57:06.699227434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1743240F70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:57:08.019713923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:08.170040225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:57:08.176321855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B179B2C47BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:57:11.861087675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:57:11.863326341Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1876F1CC31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:57:13.019650449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:13.51901026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:57:13.521525097Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B18D9C58037, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T22:57:18.01932323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:19.895180846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:57:19.897591556Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1A55D2772C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:57:22.969997096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:57:22.972390437Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1B0D17C9EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:57:23.019559915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:26.03078811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:57:26.033223195Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1BC3880D82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:57:27.665430449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:57:27.667795683Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1C24F65B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:57:28.018950325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:57:28.01897936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:57:28.018993093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:28.524992911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:57:28.527730332Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1C5837BF36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:57:28.527745963Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:57:28.527765027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:57:28.533357608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1C589117EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:57:33.019424023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:57:38.019376635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:40.102194824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:57:40.109802537Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1F0A8DAB7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:57:41.533584573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:57:41.535866321Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1F5F90EB9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:57:42.62090392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:57:42.623487926Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B1FA0659746, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:57:43.019825438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:44.513127611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:57:44.517767782Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20114DC4E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:57:45.635039525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:57:45.638763991Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20541E9148, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T22:57:45.717890777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:57:45.722365865Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20591A7A43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:57:47.332685443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:57:47.335027172Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20B93963EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:57:47.348009328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:57:47.350084841Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20BA20CC3B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T22:57:48.019263851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:48.311492006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:57:48.313755933Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B20F38F82AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:57:49.777987105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:57:49.784732039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B214B3D15A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T22:57:53.019785705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:53.791131973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:57:53.795770047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B223A50AEBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-07-02T22:57:55.208745789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:57:55.211064112Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B228EACA68A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:57:58.000898089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:57:58.019138957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:57:58.019177476Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:57:58.019191714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:57:58.589867971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:57:58.59221148Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2358338269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:57:59.800616038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:57:59.803021068Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B23A05FEC2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:58:00.985104029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:58:00.987537153Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B23E6F9E3FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T22:58:03.019831455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:03.702306068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:58:03.704593507Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2488ED2E9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:58:04.329252097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:58:04.331494842Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B24AE4AC713, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:58:06.915723419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:58:06.918223581Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B254877DAD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:58:08.019647725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:09.949416252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:58:09.951740319Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B25FD49B2B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T22:58:10.749948271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:58:10.752624057Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B262D062E98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T22:58:13.019174381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T22:58:18.019484547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:21.745168013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:58:21.752412708Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B28BCA7450F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T22:58:23.019525271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:23.442473503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:58:23.444798566Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2921883900, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T22:58:23.929702913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:58:23.932145326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B293E93DA21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:58:28.018825103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:58:28.018857466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:58:28.018856052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:28.028884262Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-07-02T22:58:28.030713668Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011592125s
level=info ts=2026-07-02T22:58:29.479192633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:58:29.483323399Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2A89752C14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:58:33.0189944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:33.573010364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:58:33.583183904Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2B7DD41126, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:58:36.008255439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:58:36.013213257Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C0EA8A295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:58:36.04018293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:58:36.042179687Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C10659C60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T22:58:36.042191014Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:58:36.042208319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:58:36.044015846Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C10832702, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:58:36.459060138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:58:36.461121584Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C295D9CD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:58:36.632659465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:58:36.634817565Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C33B6E2B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:58:37.718260919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:58:37.72060539Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2C746F04A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T22:58:38.018850244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:38.581621911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:58:38.584269361Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2CA7E51217, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:58:38.584283649Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:58:38.584301027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:58:38.586638418Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2CA80A6043, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T22:58:43.019660049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:43.87912671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:58:43.881515021Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2DE3A6571F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:58:43.886943655Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9B2DE3FCAEA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T22:58:44.913710604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:58:44.915951598Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2E2150129E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:58:44.915965843Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:58:44.915983186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:58:44.917773297Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2E216DC2A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T22:58:48.01913498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:48.446433353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:58:48.448821204Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2EF3E14CB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:58:49.203130433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:58:49.205594741Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F20FCC335, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:58:49.205612301Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:58:49.2056302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:58:49.207556158Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F211E4C65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:58:49.277646118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:58:49.27971327Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F2569BBBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:58:50.216980322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="8.6 kB" avg_uncomp="8.6 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-07-02T22:58:50.219398474Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F5D6A45EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-07-02T22:58:50.219413904Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:58:50.219432341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T22:58:50.223501835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F5DAB2E23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T22:58:50.70274374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:58:50.707110859Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B2F7A7E7CD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:58:53.018981722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:53.761577999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:58:53.763762148Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3030AFDFEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:58:55.336062276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:58:55.338311694Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B308E888D59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:58:57.089269597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:58:57.09142878Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B30F7085B92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T22:58:57.466414175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:58:57.469197913Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B310D8B01F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:58:58.00060755Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:58:58.005823227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:58:58.010680316Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B312DCECCC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:58:58.018700263Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T22:58:58.018761659Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:58:58.018786275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:58:58.018818136Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-02T22:58:58.0188263Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=warn ts=2026-07-02T22:58:58.018820012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:58:58.018846562Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.828µs success=true
level=info ts=2026-07-02T22:59:01.232140039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:59:01.234524466Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B31EDFA04C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T22:59:03.018979491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:04.449020261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:59:04.451375898Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B32ADB7E933, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:59:04.45139145Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:04.451409146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:59:04.453311687Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B32ADD79197, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:59:07.143893761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:59:07.146280237Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B334E57C721, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:59:07.146298121Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:07.146316735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:59:07.148326695Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B334E7992E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T22:59:08.019306841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:08.170908781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T22:59:08.175596046Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B338BB30FD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T22:59:10.713702137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T22:59:10.719360244Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3423522FB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T22:59:13.013627265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T22:59:13.017384077Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B34AC4A6C0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T22:59:13.01939942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:16.99739014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:59:17.00221355Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3599CE1781, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T22:59:17.012194306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:59:17.014251792Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B359A865FB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:59:17.014264535Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:17.014280536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:59:17.015943633Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B359AA238B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T22:59:17.338880378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:59:17.341069488Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B35AE007631, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T22:59:17.341082186Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:17.341098129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:59:17.342823085Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B35AE1E01E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T22:59:18.019451498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:18.577256063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T22:59:18.579628046Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B35F7D206B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T22:59:18.68893178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T22:59:18.690919237Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B35FE7638DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T22:59:20.542095025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:59:20.544441481Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B366CF04046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:59:21.317668639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T22:59:21.319849918Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B369B28BBF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T22:59:22.176071604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T22:59:22.178316693Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B36CE52AE2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T22:59:23.019580354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:23.044738612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T22:59:23.046912804Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B370218FD53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T22:59:26.060416541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T22:59:26.068172565Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B37B62CFE98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T22:59:27.398259244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:59:27.404222472Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3805CDD92E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:59:27.404240952Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:27.404262739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:59:27.406180489Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3805EE245D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T22:59:28.018842154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:28.018871185Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:59:28.018885962Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T22:59:30.376311364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:59:30.378783042Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B38B719F261, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:59:30.378804316Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:30.37882198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:59:30.380749045Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B38B73B1DA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T22:59:30.646532572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:59:30.64868392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B38C731AD75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:59:32.67739163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T22:59:32.679791271Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B394041202A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T22:59:33.019596279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:33.880935795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T22:59:33.885394126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B39881D0939, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T22:59:34.616524048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T22:59:34.621368007Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B39B3FD6223, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T22:59:38.01906238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:38.232252611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:59:38.234521383Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3A8B5A2713, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:59:38.234533866Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:38.234552064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:59:38.236371158Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3A8B7870F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:59:41.40318993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:59:41.405525556Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3B485BBD79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:59:41.405542441Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:41.405560822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T22:59:41.407382755Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3B487A818E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T22:59:43.019170519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:44.078354067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T22:59:44.080701719Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3BE7CF5066, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T22:59:44.248180609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:59:44.250366963Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3BF1EC4F5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:59:44.25037909Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:44.250394868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T22:59:44.252269078Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3BF20B24A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T22:59:46.387284634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T22:59:46.389474947Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3C716DBB07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T22:59:48.019296811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:50.05164089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:59:50.054071208Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3D4BD936B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:59:50.054088055Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T22:59:50.05410698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T22:59:50.055839902Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3D4BF76040, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T22:59:52.851187263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T22:59:52.85353523Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3DF2B58141, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T22:59:53.019095113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T22:59:54.175262597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T22:59:54.177622202Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3E41A21AA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T22:59:56.466907013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T22:59:56.46927386Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3ECA3927FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T22:59:57.035009365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T22:59:57.037737608Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3EEC1BF69F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T22:59:57.533995771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T22:59:57.539867618Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3F0A0A301E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T22:59:58.000886797Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T22:59:58.019185334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T22:59:58.019207805Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T22:59:58.019232235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:00.483559628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:00:00.487855507Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3FB9BB0FD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:00:00.498017502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:00:00.499991961Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3FBA7BB18C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:00:01.083600925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:00:01.085825072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B3FDD661415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:00:03.019300121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:03.148942686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:00:03.153126998Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B40589C24DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:00:05.399283091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:00:05.404384315Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B40DECA6011, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:00:05.40440253Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:00:05.404421701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:00:05.406426794Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B40DEEC7F8E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:00:06.89152224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:00:06.894014605Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B413794203D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:00:08.019251488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:10.287172584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:00:10.289742654Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4201FBA9EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:00:13.018977201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:13.75404713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:00:13.757071031Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B42D0A1681E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:00:15.46987237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:00:15.472289128Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4336E37FBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:00:15.585522681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:00:15.587661002Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B433DC5470A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:00:15.587674143Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:00:15.587691189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="182 B" avg_comp="182 B" total_uncomp="18 kB" avg_uncomp="18 kB" idle=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:00:15.589595319Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B433DE38980, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:00:18.01947369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:18.761688429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:00:18.764044202Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B43FB17C260, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:00:18.936510158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:00:18.938787233Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B440582981C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:00:18.938800683Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:00:18.938819551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:00:18.940782611Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4405A27187, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:00:19.731188064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:00:19.733587612Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4434E0E4D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:00:20.505728097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:00:20.508217486Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B44630D37B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:00:22.771049443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:00:22.773583798Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B44EA143F1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:00:23.0198655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:23.973843886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:00:23.976309879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4531C36B66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:00:24.607284817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:00:24.615269656Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4557D7C6AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:00:24.657309458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:00:24.664852601Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B455AD1BF92, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:00:28.018763362Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:00:28.018805825Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:00:28.018800501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:00:33.019783965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:34.284574345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:00:34.289399593Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B47987A628B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:00:36.108313137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:00:36.110421282Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B480506F827, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:00:37.576543956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:00:37.578708706Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B485C8ABE83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:00:38.019400751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:38.931907831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:00:38.934183072Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B48AD547000, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:00:39.26780931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:00:39.269930313Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B48C158716D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:00:41.755738019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:00:41.757927698Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4955A4B55B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:00:43.019411307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:43.493034453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:00:43.495286687Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B49BD306803, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:00:44.21353147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:00:44.215678462Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B49E822C81D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:00:44.669769227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:00:44.672376044Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4A035A76D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T23:00:48.019643978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:51.509279747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:00:51.515865346Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4B9B4290A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:00:52.432388435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:00:52.434689626Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4BD2066D36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:00:53.01948505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:57.198480097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:00:57.202749223Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4CEE38F3B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:00:57.202768057Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:00:57.202788521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:00:57.209168143Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4CEE9ADBA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:00:58.000568116Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:00:58.018872198Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:00:58.018883916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:00:58.018894853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:00:58.064788923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:00:58.069207225Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4D21DCD3AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:00:58.503496075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:00:58.506113829Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4D3BE36591, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:00:58.910123372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:00:58.91233523Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4D541DC1EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:01:01.175700133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:01:01.178088633Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4DDB2ACAAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:01:01.739577808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:01:01.741843689Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4DFCC49433, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:01:03.019207947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:04.573488639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:01:04.575770254Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4EA5AF4ADC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:01:04.93869729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:01:04.940957663Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4EBB73D837, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:01:08.018839951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:10.004185114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:01:10.008728699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4FE982B22E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:01:10.299456703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:01:10.310856304Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B4FFB8750FA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:01:13.019590616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:13.889967474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:01:13.894484731Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B50D11FCC21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:01:16.151723132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:01:16.155520417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5157E55899, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:01:18.018967973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:19.824249468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:01:19.828570274Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5232D3FC27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:01:23.019611067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:23.274493048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:01:23.276749811Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B53005A4783, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:01:26.160854525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:01:26.163563189Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B53AC6BCC80, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:01:28.019641609Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:01:28.019680228Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:01:28.01970398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:28.667981907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:01:28.672419465Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5441F622F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:01:30.540755555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:01:30.542933758Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B54B174D124, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:01:33.019854752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:35.593185922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:01:35.596051025Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B55DE9AA7DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:01:35.939676455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:01:35.941830531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B55F340BD55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T23:01:38.01968964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:38.611184261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:01:38.613403574Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B56927D69CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:01:42.20578356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:01:42.208403776Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5768C22EF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:01:43.018942088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:43.730108698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:01:43.732327389Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B57C399FA85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:01:44.932591675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:01:44.934799434Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B580B4713DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:01:45.687901089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:01:45.689975567Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B58384A1EC5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:01:46.94631071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:01:46.950040758Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B588364514C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:01:47.780263442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:01:47.786132234Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B58B5399973, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:01:47.79631327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:01:47.798136566Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B58B5F36823, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:01:48.018854101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:49.993586247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:01:49.996037925Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5938F0924F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:01:51.794537123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:01:51.796787698Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B59A447ACC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:01:52.546739873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:01:52.551031975Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B59D13D5C1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:01:53.019385537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:57.453082999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:01:57.458078164Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5AF5B72FBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:01:58.001139853Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:01:58.019516021Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:01:58.019520844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:01:58.019540798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:01:58.369194123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:01:58.376037129Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5B2C6F7946, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:02:03.01896621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:03.888189045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:02:03.893318217Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5C754971AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:02:06.41918453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:02:06.421390975Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5D0BF9CF0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:02:08.019412714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:10.31335832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:02:10.31580729Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5DF4196F82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:02:13.019655986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:13.102567435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:02:13.104873508Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B5E9A57A77E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:02:18.019306288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:22.470803886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:02:22.473190223Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B60C8BC2833, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:02:22.48318353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:02:22.485172332Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B60C9759038, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:02:23.019074953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:23.258390944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:02:23.260712635Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B60F7AD52F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:02:24.334191582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:02:24.337254337Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6137D5D360, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:02:25.134306822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:02:25.140090841Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6167B20EC7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:02:28.018791661Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:02:28.018799828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:28.018818735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:02:31.124727761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:02:31.126996623Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B62CC8AF60F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:02:33.019040858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:33.023192311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:02:33.02546794Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B633DB28C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:02:33.927160475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:02:33.929641488Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6373969C06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:02:36.5023858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:02:36.506187226Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B640D2A488E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:02:38.018823369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:39.439547103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:02:39.450075646Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B64BCA3889D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:02:39.450093164Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:02:39.450116949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:02:39.459209722Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B64BD303DBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:02:39.602724596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:02:39.607579944Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B64C6054238, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:02:42.458411771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:02:42.462985646Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6570379922, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:02:42.616450287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:02:42.620965398Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6579A546B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:02:43.019711838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:45.334531992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:02:45.336833748Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B661B84F33B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:02:47.146734582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:02:47.149165702Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B66878972A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:02:48.019509646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:49.111717178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:02:49.11390074Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B66FCA66008, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:02:49.447868901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:02:49.450092241Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6710AF98B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:02:50.911717588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:02:50.914030315Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6767F17C95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:02:53.019656173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:53.762188392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:02:53.764532249Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6811D8EA42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:02:54.797248518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:02:54.804171303Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B684FD0358B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:02:56.219291132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:02:56.223839067Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B68A46E9D02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:02:57.051603229Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9B68D5C647E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:02:58.000756933Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:02:58.019010958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:02:58.019044376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:02:58.019040247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:02:58.882127403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:02:58.884393832Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6943031026, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:03:03.019544819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:06.162875565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:03:06.165195329Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6AF4FA692D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:03:08.018858628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:10.865365532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:03:10.867669824Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6C0D45E7CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:03:13.01918747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:13.483550049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:03:13.486202281Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6CA9597AE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:03:16.164328673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:03:16.170027144Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6D49536C5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:03:17.464424312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:03:17.466742245Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6D969BF832, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:03:18.018916531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:18.907528703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:03:18.910220646Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6DECA18D65, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:03:23.01956402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:23.296845351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:03:23.299071766Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6EF23E8F7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:03:25.783287754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="7.3 MB" avg_comp="3.7 MB" total_uncomp="65 MB" avg_uncomp="32 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:03:25.790278844Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B6F86BAEFFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:03:28.019834563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:28.019851616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:03:28.019871819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:03:28.026977452Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-07-02T23:03:28.02884362Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009872651s
level=info ts=2026-07-02T23:03:29.776624824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:03:29.782338921Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7074AC0643, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:03:32.020825227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:03:32.023128195Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B70FA3C2448, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:03:32.532268205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:03:32.534469954Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7118B6F1FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:03:33.019343309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:34.430539565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:03:34.434723843Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7189FA21CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:03:37.582422675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:03:37.587130271Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7245DFE003, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:03:38.018822927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:38.166099359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:03:38.1706962Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7268A81C7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:03:38.374340876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:03:38.378726158Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B72750FC087, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:03:42.589020629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:03:42.591358209Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B737026D687, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:03:42.617532935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:03:42.619496085Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7371D6711E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:03:43.019751582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:45.072107087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:03:45.074451659Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7404279F78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:03:45.963827375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:03:45.966399456Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B743950AB55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:03:47.220188844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:03:47.222487586Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B74843085FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:03:48.01903435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:48.418804122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:03:48.421144492Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B74CBA1F63B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:03:53.019542951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:53.614313702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:03:53.616685824Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B760150950D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:03:58.000582706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:03:58.018792153Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:03:58.0188288Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:03:58.018854416Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:03:58.018862701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:03:58.018900434Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-02T23:03:58.018908411Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:03:58.018926392Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.229µs success=true
level=info ts=2026-07-02T23:03:58.607738878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:03:58.610129777Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B772AF12415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:04:03.019845549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:04.423190267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:04:04.425981269Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7885941BED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:04:07.993163356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:04:07.995946169Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B795A62C13E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:04:07.995965572Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:04:07.995987065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:04:08.001708968Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B795ABC2A40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T23:04:08.018860979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:11.985910006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:04:11.990636093Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7A487C918B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:04:12.077967322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:04:12.080068265Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7A4DD17D81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:04:12.915286391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:04:12.917567235Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7A7FBCD8DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:04:13.019030455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:15.820335258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:04:15.822643142Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7B2CE45961, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:04:16.674009873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:04:16.676389454Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7B5FC6FAEA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:04:18.019595649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:04:23.019876673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:23.378748159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:04:23.385746153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7CEFAE8E25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:04:23.448955645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:04:23.454801477Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7CF3CD88FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:04:24.668093007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:04:24.670565051Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7D3C43F4CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:04:27.26069513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:04:27.264946323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7DD6E28796, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:04:28.019307997Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:04:28.019337939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:04:28.019359559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:31.491055067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:04:31.495910366Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7ED3172B69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:04:31.758466503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:04:31.760670628Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7EE2DF9BD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:04:31.864979159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:04:31.867301463Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7EE9389C0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:04:32.884261589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:04:32.886519896Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7F25FAE8B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:04:33.019596659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:33.838083585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:04:33.841914775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7F5EECD550, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:04:34.673067203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:04:34.67780852Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7F90BE1652, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:04:36.461123127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:04:36.465573785Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B7FFB4C4854, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:04:38.019488908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:41.033829997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:04:41.0393395Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B810BE8E1D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:04:42.508264801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:04:42.51061213Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B81639DB159, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:04:43.018998268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:04:48.018822344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:50.612534428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:04:50.614974329Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8346AC8BBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:04:52.034999773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:04:52.037298417Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B839B741D88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:04:53.019789872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:04:53.185186423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:04:53.187813059Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B83E00325A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:04:53.187827953Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:04:53.187846018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:04:53.190140107Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B83E028536E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:04:54.818066955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:04:54.820814107Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B84415D4004, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:04:55.395442972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:04:55.401247544Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8463F44EE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:04:58.00042823Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:04:58.01979439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:04:58.019819122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:04:58.019833018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:05:03.019198849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:06.557092102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:05:06.560255756Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B86FD13B93E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:05:08.019879685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:08.757698526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:05:08.759875018Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B878032AFBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:05:08.766908107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:05:08.768687932Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8780BB0FD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:05:08.768699415Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:08.768714043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:05:08.770385716Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8780D58788, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:05:09.772125211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:05:09.77420769Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B87BCA7E20F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:05:10.600762685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:05:10.60294143Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B87EE0DF273, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:05:12.119399685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:05:12.121641885Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8848923F01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:05:13.01983199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:16.34266235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:05:16.350252187Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B89449E3846, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:05:18.019613622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:20.218990484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:05:20.224749709Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8A2B8E5582, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:05:23.01972354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:25.114480322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:05:25.116743265Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8B4F23F82F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:05:25.116762537Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:25.116779723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:05:25.118582574Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8B4F426421, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:05:26.415970645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:05:26.418401966Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8B9CB8F2F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:05:26.418421232Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:26.418439269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:05:26.420352502Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8B9CD97880, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:05:27.191402842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:05:27.193630027Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8BCAEF487E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:05:28.018763743Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:05:28.018794127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:05:28.018832379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:29.066044907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:05:29.070234806Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8C3AC6F84D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:05:29.454341314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:05:29.458944651Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8C51F47098, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:05:30.586045872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:05:30.588435204Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8C9546FDA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:05:30.603689263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:05:30.607858331Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8C96703C60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:05:30.607874275Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:30.607895192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:05:30.614498481Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8C96D5A2AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:05:31.627061876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:05:31.631545506Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8CD37412B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:05:31.631562167Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:31.631582253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:05:31.633331393Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8CD3923573, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:05:33.019342684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:35.991398417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:05:35.993564912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8DD773A4FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:05:35.993581054Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:35.993599132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:05:35.995395515Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8DD79166E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:05:36.632162133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:05:36.634450867Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8DFDA6EAC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:05:36.71664988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:05:36.718816506Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8E02ADED1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:05:37.683968858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:05:37.685939467Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8E3C548956, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:05:38.019294038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:38.427416062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:05:38.42931636Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8E68A4747F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:05:40.221871632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:05:40.224412023Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8ED39F4F47, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:05:43.019187882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:43.77565885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:05:43.777907857Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8FA76ED2E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:05:44.154028551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:05:44.156532711Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B8FBDFE2E0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:05:48.019109182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:48.257643962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:05:48.260368374Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B90B29B4313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:05:48.35327194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:05:48.358857174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B90B87C540C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:05:48.837743024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:05:48.841953256Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B90D5475B9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:05:49.775760641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:05:49.780367605Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B910D33219B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:05:49.78038405Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:49.780402376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:05:49.782266429Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B910D52829F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:05:49.853636915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:05:49.855681394Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9111B20F8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:05:51.624483388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:05:51.627042433Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B917B43EB12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T23:05:51.627057838Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:05:51.627077113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:05:51.629173649Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B917B665B5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:05:52.761782651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:05:52.764017187Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B91BF0A50AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:05:53.019699564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:57.07516616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:05:57.077514377Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B92C0248C2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:05:58.000989546Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:05:58.019257137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:05:58.019282106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:05:58.019299829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:05:59.087112311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:05:59.089460997Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B933810682A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:06:00.818822864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:06:00.825713868Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B939F8CA9BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:06:01.056477208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:06:01.06173429Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B93ADA168D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:06:02.803924839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:06:02.806292031Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B94159BD74E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:06:03.019623441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:07.051193786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:06:07.053589681Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9512C4ABB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:06:08.019458007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:09.076463161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:06:09.078732972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B958B7B1A5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T23:06:13.018996835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:14.559283643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:06:14.561610257Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B96D2494134, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:06:14.56162681Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:14.56164429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:06:14.563539797Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B96D268D8BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:06:15.083476759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:06:15.087442103Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B96F1A132E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:06:15.741259327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:06:15.747096319Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9718F2925D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:06:17.231795502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:06:17.233990594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B977192A9BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:06:18.019583845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:21.014027136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:06:21.016243693Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B985302420B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:06:21.016262574Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:21.016283248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:06:21.018033537Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9853207699, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:06:21.320811399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:06:21.322891084Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B98654A13B8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:06:21.322906079Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:21.322923109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:06:21.32461035Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B986567339A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:06:23.018976691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:24.326743159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:06:24.329146267Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B991878AEEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:06:25.244325095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:06:25.246589324Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B994F28AF1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:06:25.246609093Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:25.246629241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:06:25.248456152Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B994F47AD90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:06:25.680566256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:06:25.68483394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9969473458, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:06:28.018947346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:06:28.018981059Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:06:28.019009691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:28.680193173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:06:28.686216438Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A1C2AA226, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:06:28.68623308Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:28.686254417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:06:28.688085967Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A1C4C0F9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:06:29.434174031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:06:29.436332396Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A48E383AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:06:29.492506136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:06:29.494364174Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A4C5AB868, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:06:30.514272675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:06:30.516525759Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A8945C837, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:06:30.516542429Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:30.516565528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:06:30.518297428Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9A896360E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:06:32.829079492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:06:32.831630151Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9B1340826E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:06:33.018979363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:33.478395295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:06:33.480623988Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9B39F209C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:06:35.390483796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:06:35.393238254Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9BABF1DBCC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T23:06:38.019245259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:38.83351983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:06:38.838576319Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9C794DF727, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:06:38.838593777Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:38.838611869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:06:38.840469084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9C796D2B37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:06:40.272262163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:06:40.274606847Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9CCEE5627A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:06:40.300771023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:06:40.302731739Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9CD0945E7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:06:40.661553909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:06:40.663672514Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9CE6177E5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:06:43.019638092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:43.920666249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:06:43.922798663Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9DA85A2777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:06:44.50948371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:06:44.511622557Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9DCB71E737, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:06:44.555674483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:06:44.55752228Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9DCE30A620, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:06:48.018884166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:50.433392289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:06:50.435627167Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9F2C8BFC68, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:06:50.435653394Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:50.435672437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:06:50.437419877Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9F2CA90D71, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:06:53.019164373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:06:53.209435023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:06:53.211607497Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9FD20165FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:06:53.211625197Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:06:53.211652176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:06:53.213390146Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9B9FD21F21A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:06:55.814398421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:06:55.821018616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA06D89CFEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:06:56.683324169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:06:56.691282029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA0A168ABB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:06:56.767325477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:06:56.771830423Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA0A633E865, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:06:57.330764697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:06:57.3328477Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA0C7A726B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:06:57.964871974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:06:57.967059639Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA0ED73B100, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:06:58.000895257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:06:58.019133885Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:06:58.01916994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:06:58.019165419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:07:03.019844155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:03.219107926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:07:03.221600154Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA226A3D914, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:07:05.126570221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:07:05.1288597Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA29853E12B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:07:08.01981188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:09.489984799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:07:09.492257072Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA39C68721E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:07:09.758766104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:07:09.760889698Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA3AC6C16C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:07:09.810250954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:07:09.812233532Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA3AF7B80D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:07:10.060866348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:07:10.06299915Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA3BE6D17AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:07:12.324274399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:07:12.326332915Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA44555E1F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:07:13.018971833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:14.622799921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:07:14.625024071Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA4CE581AC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:07:16.678812615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:07:16.681175623Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA548E54019, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:07:17.943914454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:07:17.948010009Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA5946AD8B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:07:18.018984211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:18.456165354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:07:18.46053717Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA5B2F4BF44, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:07:19.231525788Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9BA5E0EBE878, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:07:19.426077363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:07:19.428347717Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA5ECA34DBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:07:20.542270548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:07:20.544509417Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA62F2B0E41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:07:21.95419074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:07:21.95683032Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA68356CCAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:07:23.019672723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:27.013672883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:07:27.016545453Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA7B0ED3095, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T23:07:28.018840451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:28.018862684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:07:28.018900616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:07:31.196666269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:07:31.202130857Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BA8AA69FD77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:07:33.019558445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:37.681425462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:07:37.683751872Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAA2CC03BEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:07:38.019576925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:42.389690658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:07:42.391897831Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAB45617479, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:07:42.410077169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:07:42.411912768Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAB4694B36E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:07:43.019725176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:43.452066261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:07:43.459548135Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAB85047CAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:07:47.748104503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:07:47.754215032Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAC84FF19DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:07:48.018950689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:48.034127154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:07:48.036244195Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAC95CEBBAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:07:48.963537233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:07:48.965655763Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BACCD3529C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:07:50.344437217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:07:50.346878791Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAD1F86CE41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:07:53.019618006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:07:54.696726036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:07:54.700713254Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAE230B7168, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:07:54.70073363Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:07:54.700764963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:07:54.70695463Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAE2369B502, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:07:56.701194635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:07:56.705695303Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAE9A8C4D46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:07:56.837054748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:07:56.839102534Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAEA280CD41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:07:58.00043692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:07:58.0198001Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:07:58.019823527Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:07:58.01984558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:00.185732831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:08:00.188102482Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAF6A1D859F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:08:00.563132369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:08:00.565300444Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BAF809A0CF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:08:03.01910384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:04.164348938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:08:04.166861269Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB05742D9EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:08:04.260181036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:08:04.262570572Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB05CF814F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:08:04.630505368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:08:04.632600885Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB073080A6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:08:05.693594743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:08:05.69780904Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB0B27E0682, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:08:06.473981958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:08:06.478909024Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB0E1130345, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:08:08.019422238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:10.762784593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:08:10.76527482Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB1E08F0C8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:08:11.901197955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:08:11.903658675Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB22469A455, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:08:13.019794071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:14.92694569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:08:14.931323022Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB2D8DFE776, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:08:18.019624944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:19.109820184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:08:19.114838175Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB3D23B7F5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-07-02T23:08:23.019076248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:27.08435373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:08:27.089076213Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB5AD8945C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:08:27.339659866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:08:27.341717282Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB5BC99A41E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:08:28.019390253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:28.019424994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:08:28.01944011Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:08:28.027039961Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-07-02T23:08:28.028885708Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009838079s
level=info ts=2026-07-02T23:08:28.957769568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:08:28.961533773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB61D250C35, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:08:32.218329393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:08:32.223256445Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB6DF8D078B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:08:32.510857948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:08:32.512927607Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB6F0D48129, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-07-02T23:08:33.019679773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:34.024653206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:08:34.02711249Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB74B11E872, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:08:35.997373505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:08:35.999559959Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB7C0A58500, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:08:37.399431975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:08:37.401760348Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB8143916CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:08:38.01882161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:40.409387315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:08:40.416121208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB8C7E3EB9B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:08:41.240329964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:08:41.245038719Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BB8F94B8E93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:08:43.019217307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:08:48.019020285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:49.892604746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:08:49.894997039Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBAFCDEC483, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:08:50.271256675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:08:50.273759184Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBB13710814, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:08:51.389471757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:08:51.391867651Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBB56176A0F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:08:53.018997712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:54.528488573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:08:54.530894313Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBC11315F78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:08:57.714401556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:08:57.716722902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBCCF156DEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:08:58.001000473Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:08:58.019243432Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T23:08:58.019267666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:08:58.019271536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:08:58.019294895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:08:58.019367008Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-02T23:08:58.019375544Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:08:58.019394902Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.875µs success=true
level=info ts=2026-07-02T23:08:58.025437759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:08:58.027728117Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBCE19E4DFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:08:59.640544426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:08:59.642909131Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBD41E42D21, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:09:01.742532188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:09:01.745270176Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBDBF30688C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:09:03.01935027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:05.315720072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:09:05.318019422Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBE94283110, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:09:08.0198235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:08.459538585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:09:08.464090103Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BBF4FABDD51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:09:13.019194528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:16.06048471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:09:16.066254872Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC114CDE0BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:09:18.018803267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:18.319436813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:09:18.322143855Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC19B438ACD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:09:20.004673203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:09:20.010724979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC1FFE83272, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:09:20.546962007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:09:20.551709869Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC220274D26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:09:22.540731723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:09:22.542961163Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC296D7EF5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:09:23.019724425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:24.330194248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:09:24.332549728Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC30182175A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:09:24.755657757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:09:24.758219621Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC31ADF2A6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:09:28.019558694Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:09:28.019589843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:09:28.019685049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:28.303387492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:09:28.305594109Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC3EE529E79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:09:30.763427907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:09:30.765855953Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC480F57062, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:09:33.019639169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:34.209071492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:09:34.211446871Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC54E558085, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:09:38.019008289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:38.020086293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:09:38.026862195Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC631C126A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:09:38.026879069Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:09:38.02689902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:09:38.037243509Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC632597BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:09:39.335408563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="130 B" avg_comp="130 B" total_uncomp="48 kB" avg_uncomp="48 kB" max_age=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:09:39.340337655Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC68009F7F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:09:39.636187031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="579 kB" avg_comp="579 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" max_age=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:09:39.639309835Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC691DE5EA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:09:39.93566278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="300 kB" avg_comp="300 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" max_age=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:09:39.945375778Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC6A41B51E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:09:40.236339673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="494 kB" avg_uncomp="494 kB" max_age=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:09:40.241258645Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC6B5BEAD01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:09:42.056664456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:09:42.058964563Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC72215B2A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:09:42.132169418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:09:42.134176579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC7269366AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:09:43.010028906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:09:43.012202313Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC75AE8284F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T23:09:43.019185458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:43.343055863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:09:43.345138322Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC76EC10E39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:09:45.491864267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:09:45.494297999Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC7EED87939, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:09:48.019391324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:48.220999246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:09:48.223206984Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC8918236BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:09:48.663792095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:09:48.668062564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC8AC061B61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:09:48.902691988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:09:48.907190977Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC8BA46D7E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:09:50.005534858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:09:50.012581039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC8FC2A98E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:09:51.120189884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:09:51.125839771Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BC93E850465, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:09:53.019067372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:57.430140856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:09:57.435137277Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCAB693BF46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:09:57.438143509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:09:57.440028142Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCAB6E1CCAE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:09:58.000840074Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:09:58.019071168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:09:58.019093633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:09:58.019137042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:09:58.335182815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:09:58.337316937Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCAEC5B4BC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:09:58.565516925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:09:58.567546855Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCAFA154633, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:09:59.250483783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:09:59.253686605Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCB22FA2653, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:09:59.452184421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:09:59.456836086Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCB2F16269A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-07-02T23:10:00.016570871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:10:00.022279306Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCB50C47394, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:10:03.019212179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:07.794691616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:10:07.797021904Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCD203230E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:10:08.019359905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:09.59920852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:10:09.603188987Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCD8BD8E830, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:10:11.439380326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:10:11.445823588Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCDF9AF1411, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:10:12.212054165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:10:12.219770293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCE27D09515, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:10:13.019596398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:10:18.01926356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:18.591087664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:10:18.595734498Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCFA3DA50D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:10:19.266458464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:10:19.268742011Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCFCBF7890C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:10:19.385969485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:10:19.387995609Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BCFD314421D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:10:21.450713476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:10:21.453210224Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD04E2AF5C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:10:22.017931745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="933 B" avg_comp="933 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:10:22.020550828Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD06FFB22F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-07-02T23:10:23.019477793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:24.716842938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:10:24.720051832Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD110E3A1D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:10:27.327044613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:10:27.332291646Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD1AC960EAF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:10:27.513813088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:10:27.518399919Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD1B7AE7C63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:10:28.019512502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:10:28.019537216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:10:28.019548514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:29.072016776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:10:29.074452157Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD2146D4834, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:10:29.451401524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="314 B" avg_comp="314 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:10:29.45380246Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD22B0A64A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:10:29.457838193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:10:29.459794605Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD22B67BDB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:10:30.612549671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="118 kB" avg_comp="118 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" max_age=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:10:30.617230221Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD2705D225D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:10:33.019663526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:35.179736143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:10:35.182178695Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD38079DB09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:10:38.018961489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:40.949182217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:10:40.951627564Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD4D85CB437, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:10:41.57933006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:10:41.581756029Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD4FDEBA1C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:10:42.018553259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:10:42.020935275Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD51818BA31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-07-02T23:10:43.019025976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:43.62748895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:10:43.629906352Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD577FFE046, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:10:43.824337174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:10:43.82731937Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD583BD3330, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:10:44.369630525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:10:44.372557229Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD5A43F0587, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:10:46.212815534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:10:46.217230422Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD61237C83E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:10:47.817541966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:10:47.823493162Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD671F6A4C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:10:48.019390054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:51.703282213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=info ts=2026-07-02T23:10:51.703460947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="350 kB" avg_comp="350 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:10:51.705637643Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD7595AF408, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:10:51.706985075Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD75972061A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-07-02T23:10:51.70699718Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:10:51.707015053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="72 kB" avg_comp="72 kB" total_uncomp="913 kB" avg_uncomp="913 kB" max_age=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:10:51.709232773Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD75990098F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:10:53.019834291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:56.171395222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="171 B" avg_comp="171 B" total_uncomp="24 kB" avg_uncomp="24 kB" max_age=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:10:56.173630769Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD863AC4BC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:10:58.000758699Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:10:58.0189502Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:10:58.018953986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:10:58.018975645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:10:58.228676692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:10:58.230831824Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD8DE4ACFF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:10:59.467843731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:10:59.470011892Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BD928269887, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:11:03.019770928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:03.618885895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:11:03.62121885Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDA1F93BE63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:11:04.298093998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:11:04.300273153Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDA480EA18B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-07-02T23:11:06.028799688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:11:06.032559294Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDAAF4F47B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:11:07.757209497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:11:07.762581425Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDB1668EFD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:11:08.019071123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:09.050653026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:11:09.05285652Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDB63556B09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:11:09.980267469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:11:09.982866657Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDB9AC09C55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:11:10.603894467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:11:10.607019734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDBBFF77E15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:11:11.131527387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:11:11.366943863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:11:11.371325469Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDBED85C7DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:11:12.07247127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:11:12.074569499Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDC177105D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:11:13.019184487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T23:11:16.395797418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://loki-minio.product-kubearchive-logging.svc:9000/chunks/kubearchive/ec4fff4ad1321f8e/19f24e0c674%3A19f24e2098a%3A92ec9f82?x-id=PutObject\": readfrom tcp 10.129.0.69:57514->172.30.127.12:9000: write tcp 10.129.0.69:57514->172.30.127.12:9000: write: connection reset by peer, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:11:17.362411524Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9BDD52A16C79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:11:17.566921619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:11:17.571095912Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDD5F0EC414, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:11:18.019049983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:20.706580984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:11:20.708943098Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDE1A16AA08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:11:21.259479773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:11:21.261652498Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDE3B08725A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:11:22.265299723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:11:22.26799426Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDE77043163, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:11:22.398379995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:11:22.40308569Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDE7F10F859, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:11:23.01938672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:23.504795511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:11:23.506964981Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDEC0DD1841, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:11:24.533271088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:11:24.53552955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDEFE2AD663, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-07-02T23:11:25.392948435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:11:25.395196459Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDF31690F7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:11:26.141183213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:11:26.14339048Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDF5E01EF36, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-07-02T23:11:28.019372057Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:11:28.019401085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:11:28.019387845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:28.378396066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="134 B" avg_comp="134 B" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" max_age=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:11:28.380630473Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BDFE35C325C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:11:30.39712059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:11:30.399464421Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE05BB067AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:11:30.57671671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:11:30.578826436Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE06661DDC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:11:31.625521231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:11:31.628004819Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE0A4E8D060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:11:31.979781335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:11:31.982250038Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE0BA05B266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:11:33.019072081Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:11:38.019676199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:38.888596553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:11:38.890800193Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE255D01A57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:11:38.890815533Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:38.890834046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:11:38.89258555Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE255ED95B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:11:40.565914596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:11:40.568402319Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE2B9CC1431, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:11:40.568419125Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:40.568437035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:11:40.570480493Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE2B9ED5784, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:11:41.835665942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:11:41.840065478Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE305996E61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:11:43.019687111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:44.572767575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:11:44.578876766Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE3A8D78ABC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:11:44.578895768Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:44.578915704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:11:44.580768809Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE3A8F75867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:11:48.019725356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:48.269527109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:11:48.271729237Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE484F4619A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:11:50.555589053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:11:50.557947764Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE50D393350, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:11:51.66500477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:11:51.667426299Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE54F5A36BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:11:51.667443628Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:51.667460979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:11:51.669326697Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE54F7A1F39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:11:53.019667798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:55.489641577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:11:55.491982553Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE633523774, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:11:55.491998284Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:55.492016269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:11:55.493797058Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE6336F68CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:11:55.616179072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:11:55.618032652Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE63AD692DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:11:55.97283762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:11:55.974829757Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE650196C4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:11:56.581567343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:11:56.583830225Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE67464C18B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:11:58.000986544Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:11:58.019278731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:11:58.019304642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:11:58.019302195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:11:58.174247271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:11:58.176848967Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE6D3543F56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:11:59.418838349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:11:59.421921233Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE71D8E329F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:11:59.770549904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:11:59.776264413Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE732ABBEEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:11:59.776282008Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:11:59.776301946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:11:59.778167975Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE732CDE866, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:11:59.900666664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:11:59.903301044Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE73A3CF434, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:12:01.427560764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:12:01.430065077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE7954117E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:12:03.019382359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:03.137642007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:12:03.140434928Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE7FB3033E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-07-02T23:12:03.140454564Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:12:03.140473424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="118 B" avg_comp="118 B" total_uncomp="195 kB" avg_uncomp="195 kB" max_age=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:12:03.142577661Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE7FB568FF5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:12:04.539972526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:12:04.542201048Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE84EC19B82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:12:05.55126274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:12:05.553468932Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE88B0871BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:12:05.677834023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:12:05.68183777Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE892AE5EF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:12:08.019755172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:11.22200636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:12:11.227076388Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE9DD31D6E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:12:11.646050991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:12:11.648434131Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BE9F64FE33A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:12:12.514744238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:12:12.517218608Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEA2A185101, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=warn ts=2026-07-02T23:12:13.01962063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:13.577599897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:12:13.584553268Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEA69B61E4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:12:14.980527863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:12:14.985245163Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEABD3450BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:12:15.692508095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:12:15.694630849Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEAE77E1B76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:12:16.581578005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:12:16.584271353Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEB1C83F170, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:12:17.904099886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:12:17.908760839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEB6B768467, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-07-02T23:12:18.019192319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:18.734004162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:12:18.736301964Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEB9CC8D766, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:12:19.210668849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:12:19.212921264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEBB932F077, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:12:19.381183285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:12:19.383297826Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEBC3598866, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:12:19.708370854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="3.9 kB" avg_comp="3.9 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-07-02T23:12:19.71047791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEBD6DA763A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-07-02T23:12:19.710491872Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:12:19.710512438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="122 kB" avg_comp="122 kB" total_uncomp="782 kB" avg_uncomp="782 kB" max_age=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:12:19.713081369Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEBD6FFBEA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:12:20.012994979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:12:20.01503229Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEBE90169F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:12:21.762190926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:12:21.764649742Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEC51492CA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:12:22.326322078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:12:22.328534168Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEC72E75A22, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:12:23.019695732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:23.126016071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:12:23.130396133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BECA2B17DE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:12:23.130413321Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:12:23.130434285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:12:23.136736145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BECA313F091, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:12:26.422549827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:12:26.427706709Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BED67389607, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:12:28.019441582Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:12:28.019472415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:12:28.019486065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:28.346639593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:12:28.349449554Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEDD9C13D88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:12:31.442665806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:12:31.445790409Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEE924DD3F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:12:31.64334166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:12:31.645767221Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BEE9E3F0A3C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:12:33.019229655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:12:38.018845251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:39.843014782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:12:39.847634073Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF0871D53A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:12:40.695050215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:12:40.702631726Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF0BA0F0866, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:12:40.904184789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:12:40.906544849Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF0C63A5CE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:12:42.548046923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:12:42.550542257Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF128385722, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-07-02T23:12:43.019438354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:43.91340591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:12:43.915809684Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF179994C26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:12:47.016826706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:12:47.019210245Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF232927FAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:12:47.431286575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:12:47.433746933Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF24B48E9F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:12:48.018986717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:49.908919578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:12:49.911324754Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF2DEF4F718, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:12:53.019093987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:12:53.32562159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:12:53.328201477Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF3AA9D9348, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-07-02T23:12:53.434085415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:12:53.437906395Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF3B1294DCE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:12:53.858185406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:12:53.863549558Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF3CA84913E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:12:53.90272163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:12:53.904741994Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF3CCFE5636, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:12:55.948281112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:12:55.950790112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF446EFB9E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:12:56.752928233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:12:56.756211704Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF476EF6A9A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:12:56.957671777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:12:56.962652787Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF483411217, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:12:56.962668307Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:12:56.96268656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:12:56.964838956Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF4836449DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:12:57.12349924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:12:57.12564534Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF48CF8FFBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:12:58.000582742Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:12:58.018819922Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:12:58.01884406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:12:58.018851296Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:00.363710374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:13:00.370738524Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF54E640A8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:13:02.714167783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:13:02.719176831Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF5DA5DCDE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:13:02.719196236Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:02.719217427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:13:02.721177219Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF5DA7EF5DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:13:03.0198751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:03.211868435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:13:03.214097936Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF5F7DF5679, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:13:03.214111992Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:03.214129767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:13:03.215958418Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF5F7FD7E4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:13:04.540846388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:13:04.543319133Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF6471822F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:13:04.543334924Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:04.543352897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:13:04.545140793Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF6473702C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:13:06.01238431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:13:06.015243124Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF69ED2F627, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:13:07.157260739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:13:07.162029257Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF6E32D5B4F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:13:07.162047878Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:07.16206705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:13:07.163881129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF6E34D64DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:13:08.019326679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:08.285988017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:13:08.288330383Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF7265011E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:13:08.288347984Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:08.288365991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:13:08.290223536Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF7266FA5B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:13:11.779986254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:13:11.782354248Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF7F69287BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:13:12.605950274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:13:12.608323479Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF827CD9B46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:13:13.019317084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:13.541188995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="332 B" avg_comp="332 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:13:13.543506062Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF85F8B8B53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-07-02T23:13:13.543519745Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:13.543537735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="67 kB" avg_comp="67 kB" total_uncomp="547 kB" avg_uncomp="547 kB" max_age=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:13:13.545491196Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF85FACD63C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:13:14.569296511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:13:14.571673583Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF89CD3E590, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:13:16.575889903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:13:16.57829605Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF9146E842F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:13:17.264194994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:13:17.26853825Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF93D936A41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:13:18.019427469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:18.504529949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:13:18.511049621Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF987A3AC4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:13:18.564086522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:13:18.56649181Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF98AEFEE97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:13:19.792215916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:13:19.794502816Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF9D4247432, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:13:20.443263143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:13:20.445589516Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BF9FAF36E76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:13:21.053380393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:13:21.055720254Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFA1F50E1F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:13:21.585012025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:13:21.58767391Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFA3F020C38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:13:22.127342488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:13:22.129684513Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFA5F54567A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:13:23.019464408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:13:28.01884599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:28.018863348Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:13:28.018894921Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:13:28.02633197Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-07-02T23:13:28.028293342Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008719122s
level=info ts=2026-07-02T23:13:29.043146267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:13:29.045376313Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFBFB8A1F2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:13:29.074436823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:13:29.076891845Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFBFD681050, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:13:33.019044117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:33.851362303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:13:33.855371173Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFD1A3AC639, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:13:37.294718277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:13:37.299997841Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFDE789E710, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:13:37.300015701Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:37.300035006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:13:37.301936873Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFDE7AAE313, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:13:38.01956738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:38.041667279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:13:38.043877041Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFE13E1E36A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:13:38.175983399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:13:38.178217051Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFE1BE38EE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:13:38.605047857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:13:38.607284208Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFE3576BD1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:13:39.909705202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:13:39.912429054Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFE833C8C56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:13:40.222563229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:13:40.225056772Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFE95E39057, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:13:40.889696073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:13:40.892125022Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFEBDA5E905, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:13:41.917050636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:13:41.91947972Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFEFAE13BB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:13:43.01987145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:43.182788409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:13:43.185217874Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFF46531E0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:13:43.470019685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:13:43.472274785Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFF5770E2FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:13:44.817940826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:13:44.820394302Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9BFFA7CB3A5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:13:46.551193564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:13:46.553663592Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C000F1A3C95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:13:46.84560142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:13:46.848346695Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0020A78619, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:13:47.566193386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:13:47.569470754Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C004BA4F9AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:13:48.01898963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:49.904048163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="951 B" avg_comp="951 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:13:49.910306816Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C00D72C8F87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-07-02T23:13:49.910324776Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:13:49.910344555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="74 kB" avg_comp="74 kB" total_uncomp="698 kB" avg_uncomp="698 kB" max_age=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:13:49.91434098Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C00D765A250, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:13:50.289386176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:13:50.291751338Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C00EDE99E39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:13:51.591198675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:13:51.593654895Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C013B83270D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:13:51.62276899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:13:51.625231638Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C013D645ABD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:13:51.904921039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:13:51.907320057Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C014E34D828, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:13:52.673972092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:13:52.681749748Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C017C5E1A01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:13:53.019884955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:53.457738278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:13:53.462414318Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C01AAE71173, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:13:57.697198562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:13:57.699542102Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C02A774CAEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:13:58.001231191Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:13:58.019514846Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:13:58.019540068Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:13:58.019572862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:13:58.019575199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:13:58.019633097Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-02T23:13:58.019651893Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:13:58.019669333Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.799µs success=true
level=info ts=2026-07-02T23:13:58.90457045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:13:58.907392881Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C02EF73493C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:14:03.019757072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:03.290415714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:14:03.295415128Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C03F4FC3A4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:14:04.563258542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:14:04.567736315Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0440D4D16D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:14:08.019405095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:11.634059459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:14:11.639174834Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C05E651410A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:14:12.616864449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:14:12.619120649Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0620BB7BFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:14:13.019731449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:13.226203452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:14:13.228734584Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C064510BE13, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:14:17.595979488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:14:17.598630862Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C074986F43C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:14:18.019659144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:18.665939376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:14:18.669716835Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C078960A98D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:14:18.669736908Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:14:18.669757975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:14:18.675500978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0789B9EC9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:14:21.85841561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:14:21.862839006Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0847B3A79D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:14:21.990068991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:14:21.992410245Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C084F6C30D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:14:22.064673479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:14:22.066661338Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0853DB07FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:14:22.662177745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:14:22.66437448Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0877797C1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:14:22.710563267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:14:22.71261548Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C087A5AF77F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:14:23.019400368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:23.77518646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:14:23.777455237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C08B9D16972, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:14:28.018858645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:14:28.018872351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:28.018892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:14:29.045529376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:14:29.047769573Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C09F3F5982F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:14:29.19002749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:14:29.192142253Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C09FC9044AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:14:30.61629355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:14:30.618692692Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0A519579DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:14:30.785957571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:14:30.788194277Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0A5BB1320E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:14:32.425347465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:14:32.427719987Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0ABD6928BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:14:33.018955123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:34.347446787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:14:34.349825078Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0B2FFC9B59, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:14:34.990714583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:14:34.994881338Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0B566DEBBF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:14:35.181304769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:14:35.187945712Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0B61EEC4BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:14:35.518980457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:14:35.521327943Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0B75CF2F2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:14:35.980797329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:14:35.983357503Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0B91575E25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:14:36.427836514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:14:36.430294297Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0BABFB4E29, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:14:38.019029655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:14:43.019366083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:45.99684875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:14:45.999628352Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0DE65D282A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:14:46.579114251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:14:46.585140087Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0E094405A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:14:46.941080584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:14:46.943330657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0E1E9D5C0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:14:46.94936355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:14:46.952346694Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0E1F25E05C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:14:47.714584487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:14:47.721913068Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0E4D029DF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:14:48.019800459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:48.125092631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:14:48.13489271Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0E65A30DA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:14:51.870064841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:14:51.875273997Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0F4492F710, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:14:53.019064202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:53.508259658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:14:53.510804411Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C0FA60F7C87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:14:58.000573984Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:14:58.018777753Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:14:58.018800415Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:14:58.01882578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:14:58.677506119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:14:58.679879323Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C10DA29D794, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:15:01.397316917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:15:01.400305835Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C117C4A5C00, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:15:03.019408027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:03.759889907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="678 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" max_age=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:15:03.764348386Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1209374EF9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:15:04.259870643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:15:04.265294438Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1227149D37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:15:04.653313939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:15:04.655565933Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C123E598373, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:15:05.919938535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:15:05.922344997Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1289D929E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:15:08.019276823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T23:15:08.939970795Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9C133DB96BF6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:15:09.676091402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:15:09.678449757Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1369BC8350, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:15:10.859826512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:15:10.862027011Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C13B048F733, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=info ts=2026-07-02T23:15:12.801007519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:15:12.803361459Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1423FE74DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:15:13.019420225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:15.083858272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:15:15.08641159Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C14AC12ABD6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:15:16.157522955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:15:16.159764944Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C14EC0E8D1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:15:17.147334363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:15:17.150061605Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C152710706B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:15:18.018899461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:19.759366708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:15:19.761797152Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C15C2BF835F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:15:21.125975709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:15:21.128799401Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1614363C06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:15:23.019213462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:25.167809561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:15:25.170323443Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C17051EBCE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:15:26.586102937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:15:26.588534811Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1759A6CC50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:15:28.019414837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:28.019429398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:15:28.019455422Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:15:28.100122085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:15:28.102596926Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C17B3E54A98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:15:31.043190807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:15:31.046026754Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C186356CD4A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:15:31.439966684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:15:31.445539197Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C187B2742B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:15:31.629086325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:15:31.633478264Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C18865B7EA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:15:31.659547059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:15:31.664439187Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C18882F092A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:15:33.019411744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:35.123039946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:15:35.125774517Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C19567DB3AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:15:36.846619601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:15:36.848905166Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C19BD369085, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:15:38.018849128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:38.288357935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:15:38.290721827Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1A1325F216, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:15:38.446411815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:15:38.453769824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1A1CDF5CA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:15:41.289115799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:15:41.294610283Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1AC632700C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:15:43.019697388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:43.597768046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:15:43.60214135Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1B4FBE0C91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:15:45.256703916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:15:45.258842925Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1BB27E29DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:15:47.220181533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:15:47.222395373Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1C27864A6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:15:47.400048683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:15:47.402390715Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1C32402557, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:15:47.556643308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:15:47.558772602Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1C3B938C12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:15:48.006935231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:15:48.009035495Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1C566A6F10, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:15:48.019090248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:15:53.019397453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:53.104999571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:15:53.107182758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1D86499E0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:15:54.256098395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:15:54.258332038Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1DCAE66CDD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:15:58.000250532Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:15:58.01960721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:15:58.019613936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:15:58.019635936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:15:59.597366274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:15:59.599820379Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1F09457B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:15:59.875014765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:15:59.878879106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1F19E95A90, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:16:01.366738414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:16:01.371441555Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1F72DFE6ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:16:02.410649516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:16:02.412764982Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1FB0F2C6C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:16:03.019863614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:03.477072327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:16:03.479276196Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C1FF08345DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:16:07.217547588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:16:07.219866182Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C20CF77FB43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:16:07.219881476Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:16:07.21989916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:16:07.221794724Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C20CF982844, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:16:07.263852125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=db0cb9c162311785 immediate=false num_chunks=1 total_comp="9.3 kB" avg_comp="9.3 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"smee-client\"}"
level=error ts=2026-07-02T23:16:07.265772692Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C20D235AB6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"smee-client\"}"
level=error ts=2026-07-02T23:16:07.265783011Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:16:07.520500266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="201 B" avg_comp="201 B" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" max_age=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:16:07.522561919Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C20E1831754, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:16:08.0190563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:12.058331989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:16:12.060661476Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C21EFFFCC74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:16:13.019736062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:16.937905315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:16:16.940125986Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2312D7B251, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:16:18.018862825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:18.757557525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:16:18.76648395Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C237FB32266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:16:18.766502652Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:16:18.766524179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:16:18.776015279Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C238046605B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:16:19.743880244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:16:19.748533449Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C23BA3C14EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:16:20.257842707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:16:20.260402412Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C23D8BCDFA5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:16:22.540266853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:16:22.542789776Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2460C93379, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:16:22.808087392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:16:22.810381744Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2470BC3971, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:16:23.019607989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:26.299641019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:16:26.302014969Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2540DA2DE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:16:27.910044535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:16:27.913078621Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C25A0E12D30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:16:27.913097992Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:16:27.913118163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:16:27.917982461Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C25A12E3CE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:16:28.019338167Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:16:28.019360967Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:16:28.019378257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:28.115254237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:16:28.120023526Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C25AD332D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:16:28.157489187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:16:28.159598493Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C25AF919863, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:16:29.107755086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:16:29.110434613Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C25E83AC3ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:16:29.686866357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:16:29.689709296Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C260AC56F55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T23:16:29.689726925Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:16:29.689746124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:16:29.69409115Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C260B0BD303, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:16:30.361050797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:16:30.365858279Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2633130419, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:16:30.813190391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:16:30.817580604Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C264E012812, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:16:32.20398748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:16:32.208544746Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C26A0E90EB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:16:32.622737963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:16:32.625351605Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C26B9BEB7F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:16:33.019562593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:33.654486102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:16:33.656776544Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C26F73B4DF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:16:34.206403415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:16:34.208742037Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2718228099, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:16:35.222524226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:16:35.231179806Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C275512E9F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:16:37.837033667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:16:37.841733853Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C27F0ACC259, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:16:38.018862689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:38.158355693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:16:38.160536874Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2803ADB8EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:16:39.40219383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:16:39.404461671Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C284DD17CAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:16:40.126420381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:16:40.13003832Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C287912940F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:16:43.018984477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:43.171889936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:16:43.17684436Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C292EABC8F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:16:43.598604165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:16:43.601340319Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2947F8E5D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:16:45.515413998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:16:45.52151935Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C29BA6AA4C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:16:48.018863672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:48.928285458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:16:48.932980784Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2A85C3FA4C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:16:49.690265564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:16:49.692528182Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2AB30A314D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:16:51.180554905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:16:51.182809946Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2B0BDDB7F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:16:53.01912853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:54.11686136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:16:54.119394977Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2BBAE44954, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:16:54.725937768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:16:54.728504285Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2BDF316D2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:16:54.765533844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:16:54.767608061Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2BE1896575, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:16:56.217740419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:16:56.21995644Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2C381AAD73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:16:58.000568685Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:16:58.018811249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:16:58.018843642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:16:58.018872947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:16:59.366193455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:16:59.372931253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2CF408C35F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:17:01.618741901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:17:01.623700534Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2D7A2DC86F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:17:02.493193715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:17:02.495553396Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2DAE28AB96, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:17:03.019577893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:05.187578921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:17:05.190358807Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2E4EC74EF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:17:05.810133337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:17:05.81557281Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2E740C0D17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:17:08.019268264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:08.21876292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:17:08.223591031Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C2F03922614, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:17:13.019531392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:13.695958854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:17:13.698552356Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3049E49BE3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:17:16.756744146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:17:16.759040141Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C310053DA88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:17:18.019538477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:20.118622882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:17:20.12117892Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C31C8B892DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:17:20.126184532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:17:20.12841252Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C31C92788BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T23:17:20.12842324Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:17:20.128439402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:17:20.130455345Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C31C947CDAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:17:20.482381391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:17:20.484713562Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C31DE648A7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:17:23.019486935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:23.245403014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:17:23.248176297Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C328318717A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:17:23.248190712Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:17:23.248210727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:17:23.250650325Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C32833E9871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:17:23.715759629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:17:23.718234196Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C329F1E0BA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:17:23.718247135Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:17:23.718264823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:17:23.720253648Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C329F419ED0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:17:25.717833427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:17:25.720175102Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C331673D36B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:17:26.343857744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:17:26.347676449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:17:26.348336789Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C333BE4E95C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:17:26.353311668Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C333C320508, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:17:28.018761309Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:17:28.018798134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:17:28.018800501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:28.737517163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:17:28.739753491Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C33CA6E6153, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:17:31.463128711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:17:31.466909515Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C346CFC0627, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:17:32.972075636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:17:32.977095219Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C34C6FDB98F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:17:33.01914365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:33.379946871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:17:33.382289869Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C34DF2457CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:17:33.97158605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:17:33.973929334Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3502696BD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:17:34.118576726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:17:34.120748781Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C350B296FC9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:17:38.019127885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:40.756068937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:17:40.758594937Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3696CCD882, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:17:41.434852186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:17:41.437162913Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C36BF40841C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:17:41.554299173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:17:41.556813635Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C36C65F60E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:17:41.650141353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:17:41.652797949Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C36CC178A06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:17:43.019672356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:43.851020363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:17:43.853795055Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C374F476613, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:17:44.608975101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:17:44.611361736Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C377C721F93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-07-02T23:17:46.423841098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:17:46.426254675Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C37E89F27E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:17:48.018996913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:49.362761556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:17:49.365011869Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3897CAC405, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:17:50.779773525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:17:50.786674331Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C38EC8560A3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:17:53.01947035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:54.40093207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:17:54.405654245Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C39C43DF5F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:17:54.641185728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:17:54.643372587Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C39D2691E2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:17:55.283032825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:17:55.286050831Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C39F8B70CBC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:17:56.054191843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:17:56.059284139Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3A26C71B8F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:17:58.000481576Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:17:58.01876538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:17:58.01879319Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:17:58.018802409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:17:58.411485513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:17:58.414308299Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3AB32AF21C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:17:59.946084247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:17:59.951864798Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3B0EC71EBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:18:00.091352739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:18:00.094182039Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3B1748C84B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:18:02.678305912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:18:02.680947688Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3BB178F40F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:18:03.018968416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:06.675082589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:18:06.677653519Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3C9FB2542F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:18:07.02778041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:18:07.030611275Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3CB4B7B284, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:18:08.019799626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:08.2228653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:18:08.225296334Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3CFBF2A597, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:18:09.074598858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:18:09.076993265Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3D2EB71187, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:18:09.077009148Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:09.077026799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:18:09.078972952Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3D2ED7A496, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:18:10.191944681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:18:10.194616676Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3D7152DAB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:18:11.208554861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:18:11.211038955Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3DADE931D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:18:13.019140287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:14.117787565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:18:14.120290355Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3E5B50996F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:18:14.120307473Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:14.120331273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="147 kB" avg_uncomp="147 kB" max_age=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:18:14.122800477Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3E5B78B045, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-07-02T23:18:18.01956185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:18.181916518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:18:18.186208058Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3F4DA9C0E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:18:19.697906058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:18:19.70778468Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3FA85D6E78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:18:20.567023103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:18:20.571792758Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3FDBDBF633, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:18:20.602087023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:18:20.604424103Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3FDDCCEC30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:18:21.121874463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:18:21.124096097Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C3FFCC8CF6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:18:23.019783036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:24.415524634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:18:24.417786294Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C40C11ACAD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:18:26.334143226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:18:26.336428175Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C413376AB05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:18:26.609252413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:18:26.611464148Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4143DAAEE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:18:26.61147775Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:26.61149625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:18:26.613269324Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4143F8ABCF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:18:28.019689368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:18:28.019718463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:28.019726779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:18:28.027649558Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-07-02T23:18:28.029454908Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009659755s
level=info ts=2026-07-02T23:18:28.115372198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:18:28.117704332Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C419DA1A4A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:18:28.117716448Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:28.117732756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:18:28.119779603Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C419DC13482, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:18:28.410333912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="2.8 kB" avg_comp="2.8 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:18:28.412335848Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C41AF33579D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-07-02T23:18:28.41234812Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:28.418504172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="465 B" avg_comp="465 B" total_uncomp="238 kB" avg_uncomp="238 kB" max_age=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:18:28.420550313Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C41AFB24115, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:18:30.742426642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:18:30.744652891Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C423A3664C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:18:30.744673307Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:30.744694006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:18:30.746555188Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C423A55F2E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:18:33.019615937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:35.790795673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:18:35.793055179Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C43671F87AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:18:35.79306982Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:35.793089443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:18:35.794887735Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C43673D74CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:18:37.170255472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:18:37.172574206Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C43B958BD53, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:18:37.197883594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:18:37.199797897Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C43BAFA5AEF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:18:38.019808207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:38.349924832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:18:38.351997712Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C43FFA64DD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:18:39.30325518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:18:39.305786039Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C44387DF7E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:18:40.122452016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:18:40.124687268Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C44694E39F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:18:42.235022547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:18:42.237241796Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C44E7399728, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:18:42.315413845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:18:42.317615038Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C44EC048918, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:18:43.01902224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:44.724426954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:18:44.727120183Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C457B9F12CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:18:44.727135189Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:18:44.727172145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:18:44.72938214Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C457BC27265, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:18:44.74245724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:18:44.744487471Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C457CAAA12F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:18:44.795782745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:18:44.797995757Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C457FDA884B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:18:45.671709871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:18:45.673964206Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C45B411D418, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:18:47.542533539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:18:47.54474149Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C462393EF94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:18:48.018863704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:49.250067077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:18:49.252382808Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C46895C082A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:18:49.869013238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:18:51.12535707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C46F8FF2218, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:18:51.906701738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:18:51.911194186Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4727D5E30D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:18:53.019226714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:53.306186536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:18:53.309269016Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C477B2A7CC6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:18:53.712428936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:18:53.717396606Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C47937C33CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:18:54.278009187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:18:54.280472231Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C47B50C66C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:18:56.760182337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:18:56.763102585Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4849079632, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:18:57.412090737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:18:57.416987074Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C486FFF8F5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:18:57.708658117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:18:57.710916419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C488187A064, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:18:58.001182171Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:18:58.019452362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:18:58.019456529Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:18:58.01948722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:18:58.019506864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:18:58.019574302Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-02T23:18:58.019584878Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:18:58.019601595Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.285µs success=true
level=info ts=2026-07-02T23:18:59.326918543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:18:59.330785461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C48E213FA42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:19:00.1349779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:19:00.139887953Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C49124D2955, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:19:00.139905119Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:00.139923099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:19:00.1417728Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C49126B909A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:19:00.57076944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:19:00.573501614Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C492C21B0A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:19:02.126859127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:19:02.129429048Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4988E02234, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:19:02.12944558Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:02.129463534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:19:02.131325289Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C498901ADDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:19:03.01911411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:04.456799184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:19:04.461367267Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4A13E0C1D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:19:04.461385181Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:04.461408891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:19:04.468188933Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4A144A9DA0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:19:05.257569251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:19:05.262487905Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4A439FF8B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:19:07.324825788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:19:07.327507038Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4ABEB5245B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:19:07.327523562Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:07.327545674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:19:07.329693453Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4ABED9E072, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:19:07.61453212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:19:07.616807128Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4ACFF68F88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:19:08.018846827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:10.028472653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:19:10.031471445Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4B5FDC4ADE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:19:11.373357688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:19:11.375814483Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4BB003B34F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:19:11.3758313Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:11.375848847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:19:11.377855398Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4BB02445FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:19:13.019756088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:15.213753251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:19:15.216244027Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4C94EB5443, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:19:17.592707525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:19:17.595234667Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4D22B77161, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:19:18.019479583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:18.857035261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:19:18.859723124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4D6E1430BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:19:19.655002543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:19:19.657348871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4D9DA24832, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:19:21.636703163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:19:21.64131392Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4E13E1A1D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:19:22.481073746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:19:22.487343879Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4E46506378, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:19:23.019185079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:25.467440422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:19:25.469658687Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4EF81364DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:19:26.251505568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:19:26.253735928Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4F26CFF384, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:19:26.388988546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:19:26.39145791Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C4F2F029EC4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:19:26.650810191Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9C4F3E7D3706, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T23:19:28.019645265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:28.019656707Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:19:28.019682447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:19:30.324879829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:19:30.327336163Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C50199D34A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:19:30.327354178Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:30.327377205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:19:30.329306986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5019BD0C1C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:19:30.799746669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:19:30.802067389Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5035EA2A0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:19:30.802081844Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:30.802100435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:19:30.803863766Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C503606EB2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:19:31.296190966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:19:31.29846531Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C50537FF0C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:19:32.241018991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:19:32.243431892Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C508BD1E87A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:19:33.019070873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:34.845560854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:19:34.848045232Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5127105536, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:19:36.465457536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:19:36.46771076Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C51879CFF75, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:19:38.018939671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:38.629516361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:19:38.631976491Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C52089B108B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:19:40.892703496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:19:42.084368958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C52D661F769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:19:42.831278265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:19:42.835569972Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C53032AD10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:19:42.835589225Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:42.835607857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:19:42.837400928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5303492C82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:19:43.019290815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:43.142370783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:19:43.145017156Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C53159C8C72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:19:44.389654718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:19:44.39544014Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C536021B988, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:19:44.459483538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:19:44.463737161Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C53643746CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:19:44.877693762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:19:44.879815614Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C537D030056, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:19:46.839464563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:19:46.841724416Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C53F1F45FAC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:19:48.019627601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:48.125938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:19:48.128193941Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C543EA1282D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:19:48.345604211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:19:48.347835489Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C544BB981E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:19:49.374766543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:19:49.379237826Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C548931F5FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:19:50.468961729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:19:50.473756746Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C54CA6F35B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:19:50.473777618Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:19:50.473797398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:19:50.475595113Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C54CA8E9BA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:19:52.439954972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:19:52.442559455Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C553FC770C7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:19:52.507856195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:19:52.511762144Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5543E9B99E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:19:52.56479948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:19:52.569322131Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5547568119, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:19:53.019763404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:54.299339164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:19:54.30231119Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C55AE9E32D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:19:55.129713507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:19:55.13232894Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C55E019E5C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:19:58.000454366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:19:58.019825656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:19:58.019833241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:19:58.019866579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:19:59.206529195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:19:59.209033032Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C56D317E17F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:20:00.141509088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:20:00.144350779Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C570AD4E5EC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:20:03.01955793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:04.04140804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:20:04.043890836Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C57F345EB8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:20:04.04391013Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:20:04.043928801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:20:04.045998717Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C57F3689552, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:20:04.510272549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:20:04.512794264Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C580F38B828, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:20:04.956767031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:20:04.959780938Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5829D72FB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:20:08.019558005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:08.135836361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:20:08.13841336Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C58E75233A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:20:13.019587184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:14.101653576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:20:14.104319886Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5A4AEAB042, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:20:15.175099008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:20:15.177804019Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5A8AE6784E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:20:18.019556595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:19.00140452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:20:19.004002294Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5B6EF5A15F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:20:19.586325778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:20:19.588766634Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5B91D1C715, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:20:20.037193754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:20:20.040338506Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5BACB588D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:20:23.019844999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:23.167934123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:20:23.170378303Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5C674CA92B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:20:24.779242323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:20:24.782197692Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5CC75A0584, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:20:24.782215193Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:20:24.782236971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:20:24.784764606Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5CC7825A97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:20:25.180095757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:20:25.182688592Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5CDF3DE61A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:20:28.019786841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:28.01979986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:20:28.019822403Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:20:29.535318566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:20:29.537951427Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5DE2D656DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:20:30.116742865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:20:30.123731819Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5E05C0CB30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:20:31.228673175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:20:31.235321832Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5E48005CEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:20:33.019853358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:33.383191882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:20:33.386556633Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5EC83B1DE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:20:33.998053702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:20:34.002696416Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5EECF59252, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:20:37.346730179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:20:37.349071356Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C5FB46B4418, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:20:38.019678287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:38.86370056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:20:38.866436493Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C600ED72AA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:20:38.896616755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:20:38.898721802Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6010C945BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:20:40.069034577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:20:40.071518837Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6056AED26B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:20:40.543753371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:20:40.548129792Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C60731930BF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:20:40.664410034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:20:40.669318575Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C607A5292D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:20:43.01908344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:43.227293745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:20:43.231786868Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C61130F87EB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:20:43.88050197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:20:43.882803335Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6139DD68C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:20:44.660027705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:20:44.662633325Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C616855071F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:20:44.978362546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:20:44.980519446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C617B4BEE7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:20:45.039000548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:20:45.041177592Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C617EE8FB2D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:20:45.406344202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:20:45.40854978Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6194CF2A04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:20:46.636846515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:20:46.63919086Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C61DE27D4F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:20:48.018853665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:49.329922992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:20:49.333857903Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C627EC62992, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:20:49.333876622Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:20:49.333897901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:20:49.339721206Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C627F1F7B0E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:20:49.557281254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:20:49.562015078Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C628C5D3AD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:20:51.287789803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:20:51.290044279Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C62F35E5C32, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:20:53.01942864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:54.925278692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:20:54.927540958Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C63CC2EC707, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:20:58.000503323Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:20:58.018865221Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:20:58.018867586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:20:58.0188882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:21:00.537333013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:21:00.539586849Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C651AB094DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:21:03.018994644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:03.02613286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:21:03.028412621Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C65AF0760FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:21:08.019085272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:08.547177597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:21:08.551006107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C66F8335BDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:21:09.389718266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:21:09.393963407Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C672A73452D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:21:12.027870116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:21:12.030507898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C67C7964466, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:21:13.01897447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:15.412890139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:21:15.415075113Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6891566415, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:21:15.778757538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:21:15.781036361Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C68A7256351, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:21:16.697943631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:21:16.700216281Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C68DDEEDA17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:21:18.019608011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:19.215749369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:21:19.21817767Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6974027059, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:21:21.818404019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:21:21.820945481Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6A0F2576F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:21:22.964249069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:21:22.966398647Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6A536D7032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:21:23.019256526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:23.649458929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:21:23.65183506Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6A7C47336D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:21:23.893520444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:21:23.897811492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6A8AF19253, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:21:23.90482455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:21:23.910278102Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6A8BAEFDA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:21:24.859935838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:21:24.862916744Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6AC478AEB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:21:28.019464816Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:21:28.019502136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:21:28.019505518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:29.902055551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:21:29.911832445Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6BF166B1CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:21:32.324440292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:21:32.330167947Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6C818B3245, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:21:33.019879398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:33.795020405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:21:33.79966837Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6CD923A9D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:21:34.185501068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:21:34.187994188Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6CF048BA6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:21:35.000447861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:21:35.00277892Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6D20D9C9CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:21:35.247331247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:21:35.249598559Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6D2F90A8D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:21:36.093865893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:21:36.097876646Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6D621F4A6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:21:38.019085803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:38.377725795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:21:38.383020196Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6DEA4EDBDA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:21:38.488140074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:21:38.490321277Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6DF0B8D07D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:21:39.816842975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:21:39.81917144Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6E3FECD5A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:21:40.456751256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:21:40.459198973Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6E6612768F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:21:43.019269865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:44.04581919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:21:44.048262261Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6F3BFF540C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:21:44.4393311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:21:44.441661474Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6F537244ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:21:45.967458295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:21:45.969723431Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C6FAE873AE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:21:48.019654361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:21:53.019234398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:58.000274343Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:21:58.019626932Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:21:58.019653642Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:21:58.019657687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:21:58.317667099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:21:58.320824521Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C728EB5EEB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:21:59.417535703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:21:59.420337274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C72D03A20C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:22:00.427027683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:22:00.429387781Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C730C640BB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:22:01.12724985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:22:01.129940898Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C733621E0F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:22:02.393640345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:22:02.396068654Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C73819C8B62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:22:03.019335474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:06.681515329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:22:06.68398754Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C748130B49C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:22:08.019377053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:08.248182429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:22:08.250839082Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C74DE91ABD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:22:08.836402184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:22:08.840855491Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7501C125AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:22:10.373768512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:22:10.380337512Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C755D81BB2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:22:11.274023491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:22:11.277057767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7592F69996, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:22:13.01907728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:13.285698675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:22:13.29041396Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C760AF836F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:22:14.638342726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:22:14.640700165Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C765B750839, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:22:17.422165364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:22:17.424554793Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C770162DF70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:22:18.019407146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:19.655948952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:22:19.658343786Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7786879B11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:22:20.994946322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:22:21.001826337Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C77D69C38D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:22:21.589186569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:22:21.593891904Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C77F9E677A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:22:21.686060826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:22:21.690531549Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C77FFA6EB1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:22:22.802431017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:22:22.804564834Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C78420FBAFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-07-02T23:22:23.019013821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:24.474944534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:22:24.477575315Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C78A5C449D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:22:24.503734065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:22:24.50575562Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C78A776FB87, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:22:25.76873864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:22:25.771086031Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C78F2E093CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:22:26.348741694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:22:26.350928141Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C791570CC1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:22:27.368064195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:22:27.370264149Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C795231FA4D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:22:27.602633467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:22:27.606330044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C796043E553, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:22:28.01876733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:22:28.018792028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:22:28.018829744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:31.609289361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:22:31.613971353Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7A4F24499C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:22:33.019100582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:37.2420121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:22:37.24436053Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7B9EBCED07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:22:37.250388435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:22:37.252328242Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7B9F388BED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:22:38.018951556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:40.170703313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:22:40.173181926Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7C4D4F91C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:22:41.257193667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:22:41.259592291Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7C8E107E56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:22:43.019062263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:44.420127794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:22:44.422584423Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7D4A97D5A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:22:48.018957302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:48.20058002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:22:48.202885295Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7E2BEB5A48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:22:51.677093897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:22:51.679912944Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7EFB262703, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:22:53.019584338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:53.138503854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:22:53.14116324Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C7F523F35F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:22:58.000241076Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:22:58.018865835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:22:58.018884102Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:22:58.018891581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:22:59.056603174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:22:59.059335158Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C80B3005901, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:23:00.405472856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:23:00.408618364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C81036FE457, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:23:00.408639327Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:23:00.408660451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:23:00.414357116Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8103C8543C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:23:01.429978866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:23:01.434700893Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C81409801DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:23:03.01886997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:05.736739986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:23:05.739083528Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C824127371B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:23:05.985637833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:23:05.987837227Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C824FFC23D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:23:06.484375761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:23:06.489982344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C826DE80B07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:23:08.018813268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:09.741053486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:23:09.747348254Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8330113655, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:23:12.038665434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:23:12.04354857Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C83B8EE2588, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:23:12.043567205Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:23:12.043587506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="190 B" avg_comp="190 B" total_uncomp="83 kB" avg_uncomp="83 kB" max_age=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:23:12.045671151Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C83B9114FD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:23:12.711449237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:23:12.714187883Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C83E0E7960C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T23:23:12.714211342Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:23:12.714238422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:23:12.719338558Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C83E1368045, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:23:13.018995074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:13.455469801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:23:13.460237584Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C840D5E538D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:23:14.138863549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:23:14.141371463Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8435F81F07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:23:14.141386816Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:23:14.141405907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:23:14.14333808Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C843618C010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:23:15.32370108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:23:15.326012771Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C847C947552, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:23:16.108251936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:23:16.11085951Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84AB59BF73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:23:16.250194707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:23:16.253010245Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84B3D71B85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:23:16.620891709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:23:16.625109444Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84CA037104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:23:17.139258433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:23:17.141481954Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84E8CAD7F3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:23:17.434326829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:23:17.436741173Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84FA61E7F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:23:17.476779454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:23:17.478652721Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C84FCE5891E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:23:18.019323247Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:18.140754169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:23:18.147281591Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8524BE69EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:23:18.147294203Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:23:18.147313874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:23:18.155820674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C852542526E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:23:21.32967338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:23:21.334332654Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C85E2B39928, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:23:22.809385533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:23:22.81158717Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C863AC19597, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:23:23.01933806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:25.509738268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:23:25.512014995Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C86DBB64ED5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:23:27.931184309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:23:27.933457821Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C876C0AC759, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:23:28.019411372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:23:28.019410017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:28.019436657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:23:28.027927297Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-07-02T23:23:28.029916918Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010162225s
level=error ts=2026-07-02T23:23:28.151777871Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9C877911D287, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:23:28.289848876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:23:28.291899818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8781694826, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:23:29.192249313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:23:29.194800759Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C87B73AE6B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:23:30.861276184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:23:30.866082993Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C881AD7862C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:23:31.63725211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:23:31.639494036Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8848F201E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:23:32.914186644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:23:32.916435442Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C88950D7ADC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:23:33.019501481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:34.01051567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:23:34.01285143Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C88D6676126, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:23:37.061802196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:23:37.068896915Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C898C8E7F42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:23:38.019186993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:39.623537028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:23:39.628332265Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8A251B8286, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:23:40.693242595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:23:40.695880432Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8A64BB34B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:23:43.019241197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:46.434320165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:23:46.436782428Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8BBAED4F93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:23:48.018855013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:48.488770541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:23:48.491106763Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8C356075CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:23:49.400193619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:23:49.40287084Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8C6BB55B7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:23:50.306222051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:23:50.310257105Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8CA1CD0BBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:23:51.618654897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:23:51.625393375Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8CF02F2DFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:23:52.194912976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:23:52.201798548Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8D128B1976, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:23:53.019771373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:53.697728435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:23:53.70219987Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8D6BFAE6B2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:23:56.234041451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:23:56.236419809Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8E0307F8AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:23:57.41996131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:23:57.422308058Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8E49B7363C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:23:57.764969495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:23:57.767522627Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8E5E47C944, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:23:58.000278161Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:23:58.019655733Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:23:58.019660372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:23:58.01966613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:23:58.019680822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:23:58.019783462Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-02T23:23:58.019793366Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:23:58.019813863Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.994µs success=true
level=info ts=2026-07-02T23:23:58.880423953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:23:58.882740473Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8EA0C3910A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:23:59.267874142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:23:59.270667301Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8EB7E3A675, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:23:59.863160354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:23:59.86841174Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8EDB81A145, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:24:00.808287975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:24:00.810753584Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C8F13AE07DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:24:03.01990795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:05.256640266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:24:05.259060481Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C901CD233C1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:24:05.294180309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:24:05.296290747Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C901F0C2DBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:24:07.711040773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:24:07.713753005Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C90AF1FB108, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:24:07.71376825Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:07.713787613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:24:07.715710677Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C90AF42D282, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:24:08.019631219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:09.619988373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:24:09.622382782Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9120E5911F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:24:09.880063341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:24:09.882382659Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91306524E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:24:09.882397235Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:09.882414676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:24:09.884429631Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9130850003, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:24:12.508669263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:24:12.511014118Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91CD12CDC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T23:24:12.511030439Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:12.511047898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:24:12.512861722Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91CD3185B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:24:12.690355443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:24:12.726144565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:24:12.730664006Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91DA2B1EA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:24:12.759276411Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91DBE0DC24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:24:12.940260196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:24:12.944571182Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91E6E9F0B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:24:13.019832411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:13.22038062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:24:13.222540146Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C91F77B1B40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:24:14.489965789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:24:14.4927946Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C92432C6330, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:24:14.492812099Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:14.492832352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:24:14.496084648Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C92435EBF3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:24:18.019005029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:19.515689777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:24:19.518171771Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C936EB92BAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:24:21.458277502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:24:21.460933622Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C93E283B4D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:24:22.613869352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:24:22.616549776Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9427648642, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:24:23.01907489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:23.940830907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:24:23.943095208Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C947679B5BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:24:26.176613703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:24:26.178949671Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C94FBBDEABE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:24:26.836852085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:24:26.839265517Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C952318D83A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:24:28.018760783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:24:28.018790436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:24:28.018805094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:28.620090268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:24:28.622318126Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C958D612693, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:24:29.843501654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:24:29.846276749Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C95D64FE6C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:24:33.018993839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:37.953182132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:24:37.957626508Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C97B9CE437F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:24:38.019517446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:39.682968995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:24:39.691013161Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C98211FFCEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:24:41.215322425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:24:41.220003568Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C987C4132E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:24:43.019833246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:43.035960893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:24:43.038582971Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C98E8A60311, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:24:43.37789579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:24:43.380081143Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C98FD03D403, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:24:43.915709301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:24:43.91796933Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C991D12F7D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:24:45.043333978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:24:45.045541967Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C99604902EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:24:45.045557594Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:45.045575948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:24:45.047335142Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C996065DC6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:24:45.249012391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:24:45.251381346Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C996C8B7CA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:24:48.019755926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:48.410499885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:24:48.412675338Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9A28FAEF02, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:24:49.41904988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:24:49.422091369Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9A65250938, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:24:49.853275728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:24:49.857689016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9A7F1BF09D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:24:51.88546029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:24:51.887930856Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9AF81D042E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:24:51.887949925Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:51.887969964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:24:51.890114888Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9AF83E9289, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:24:53.019020454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:53.134275642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:24:53.136709212Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9B428C1189, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:24:54.479094328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:24:54.481264426Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9B92B2D850, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:24:54.48127868Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:54.481296635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:24:54.483039735Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9B92CF34C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:24:55.717381995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:24:55.719637143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9BDC82DBE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:24:57.746779485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:24:57.749465152Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9C557F21AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:24:58.00113406Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:24:58.010373076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:24:58.014802901Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9C654F9079, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:24:58.014816372Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:24:58.014832919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:24:58.016657618Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9C656E7E7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:24:58.019815121Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:24:58.01982751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:24:58.019854164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:25:00.510346372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:25:00.51276563Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9CFA340CED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:25:00.978762224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:25:00.981090479Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9D161D43F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:25:02.477365057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:25:02.479798349Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9D6F7233C0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:25:02.790715215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:25:02.793334982Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9D821E7B38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:25:03.019705335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:03.576177141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:25:03.578514949Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9DB0EE1BCD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:25:03.709244958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:25:03.715997733Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9DB921A8DE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:25:04.490106224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:25:04.495121549Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9DE7917E64, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:25:05.153716383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:25:05.155938936Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9E0EF5523A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:25:05.579009166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:25:05.581632566Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9E2850C2E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:25:08.004701203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:25:08.007187306Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9EB8E646A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:25:08.00720208Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:08.007219186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:25:08.009051848Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9EB90609F6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:25:08.01918852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:10.253396274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:25:10.255822484Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9F3EEEB27E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:25:10.347024868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:25:10.349394557Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9F44819C74, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:25:13.019859584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:13.162723077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:25:13.165058026Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9FEC564417, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:25:13.165071787Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:13.165088751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:25:13.166979312Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9C9FEC75D7DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:25:15.704975209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:25:15.707415432Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA083DEB71E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:25:18.019568341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:18.763492653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:25:18.767994258Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA13A4BEB9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:25:18.768013412Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:18.76803646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:25:18.776386185Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA13ACBE7A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:25:19.13349344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:25:19.13790718Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA15059CA81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:25:20.505716432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:25:20.508029941Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA1A2033DD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:25:23.019761213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:23.446910677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:25:23.449702523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA25155E155, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:25:23.449719274Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:23.449738634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:25:23.452145274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA2517B24AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:25:25.360195893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:25:25.362611454Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA2C35D6772, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:25:25.92617977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:25:25.928775059Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA2E51AF4AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:25:27.880414556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:25:27.882675792Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA359939777, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:25:28.018767715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:25:28.018800035Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:25:28.018835918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:28.039063863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:25:28.0416207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA363096E8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:25:30.8188529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:25:30.821320649Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA408BBC05E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:25:32.295980129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:25:32.300882433Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA460EC2A2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:25:32.438038975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:25:32.44276886Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA4695F5976, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:25:33.019793321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:36.743468185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:25:36.745716551Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA569DAFEB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:25:38.019867389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:38.91784878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:25:38.921674101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA5EB8C338D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:25:39.510259268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:25:39.514868696Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA60EE83337, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:25:40.086565503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:25:40.089102519Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA631204FC3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:25:40.236572725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:25:40.238817986Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA63A0F43E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:25:40.294986613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:25:40.298145366Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA63D98612A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:25:40.298174198Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:40.298192501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:25:40.299898468Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA63DB58E04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:25:40.825385934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:25:40.827963028Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA65D29A20E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:25:43.019136976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:45.651389359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:25:45.653665112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA77CCEF839, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:25:47.400519666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:25:47.403142962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA7E512F1B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:25:47.644632613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:25:47.646877821Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA7F39CADB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:25:48.018846215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:48.299547835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:25:48.302466973Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA81AB08BD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:25:49.392248322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:25:49.396946547Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA85BED0FFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:25:49.39696719Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:49.396987731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:25:49.398771107Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA85C0ACD25, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:25:49.96613745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:25:49.968282014Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA87DFAB070, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:25:52.232542691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:25:52.235431273Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA9051ACB86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:25:52.829473592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:25:52.833955772Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA928C83934, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:25:52.880005556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:25:52.882049084Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA92BA85683, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:25:52.882061082Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:25:52.882076316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:25:52.883795683Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA92BC47097, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:25:53.01977133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:53.399947635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:25:53.402146379Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CA94AA70708, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:25:57.643041142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:25:58.001185561Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:25:58.019479074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:25:58.019493776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:25:58.019507834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-07-02T23:25:58.272900502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAA6CF76FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:25:59.68612727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:25:59.690767983Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAAC17B2C49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:26:02.062790694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:26:02.065260605Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAB4F008BC1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:26:02.065277374Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:02.065294805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:26:02.067191628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAB4F21C61E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:26:02.962078093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:26:02.964453273Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAB849AD67A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:26:03.019583144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:03.965934714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:26:03.968339399Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CABC0710B78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:26:03.968354544Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:03.968373313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:26:03.970277155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CABC090D663, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:26:04.041813966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:26:04.046097054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CABC5145848, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:26:04.922867287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:26:04.927713092Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CABF9A20AB2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:26:08.019630319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:08.299590364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:26:08.301821306Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CACC2BF65E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:26:10.253493127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:26:10.255936191Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAD3736F86B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:26:12.027073148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:26:12.029382004Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CADA0EC9B79, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:26:12.36519548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:26:12.367682428Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CADB51542F7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:26:12.923147234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:26:12.92558325Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CADD655D10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:26:13.019512372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:13.192016428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:26:13.19453977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CADE65DED61, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-07-02T23:26:18.019343875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:18.385308217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:26:18.387576335Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAF1BE6EF8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:26:19.189271556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:26:19.191465029Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAF4BD16B9C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:26:21.826811276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:26:21.829330621Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CAFE90953BE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:26:23.019488219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:23.310246871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:26:23.312400401Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB04172D89D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:26:23.312416241Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:23.312433937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:26:23.314282476Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB041909838, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:26:25.469757116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:26:25.471949499Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB0C229E60E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:26:27.070544261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:26:27.072795248Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB12194CFFD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:26:28.019263394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:28.019352104Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:26:28.019377149Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:26:28.77511345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:26:28.777272934Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB1872D8FB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:26:28.777287531Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:28.77730482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:26:28.779013725Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB1874A2749, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:26:30.507261735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:26:30.509629901Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB1EE6E12C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:26:30.770199826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:26:30.772384624Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB1FE179156, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:26:32.470908983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:26:32.47321243Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB2637830E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:26:32.473228358Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:32.47324572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:26:32.475030466Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB263972E63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:26:33.019684228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:35.685254808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:26:35.687526324Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB3230F6C3A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:26:35.687542756Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:35.687561561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:26:35.689353569Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB3232D733A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:26:36.618051256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:26:36.621795208Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB35ABEF55C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:26:36.668817655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:26:36.674770817Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB35DE76DB3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:26:38.019786149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:38.504048983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:26:38.506337305Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB3CB11BDEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:26:39.902023807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:26:39.904232665Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB41E64FF24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:26:42.654535615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:26:42.657604056Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB4C281F018, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:26:43.019306325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:43.558033113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:26:43.562950686Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB4F876BE8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:26:44.738723754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:26:44.740974033Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB53EAEC021, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:26:45.32261515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:26:45.324966271Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB5617D8081, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:26:45.820397581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:26:45.82300023Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB57F2A6744, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:26:45.823013862Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:26:45.823031237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:26:45.825271356Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB57F4D1CF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:26:47.756193726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:26:47.758453356Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB5F289E0F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:26:48.019511634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:49.592525946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:26:49.59483983Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB65FFF2266, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:26:49.686953016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:26:49.689563789Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB665A08567, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:26:51.371195339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:26:51.373513448Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB6CA035270, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:26:51.873015214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:26:51.875747899Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB6E7F2BAE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:26:52.712809493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:26:52.717892763Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB71A2360B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:26:52.774180162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:26:52.776138529Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB71D9F6723, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:26:52.935642873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:26:52.937955551Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB727421923, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:26:53.019009289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:54.111266391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:26:54.113828334Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB76D58CE07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:26:55.570370875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:26:55.572643044Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB7C44E888B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:26:56.474208565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:26:56.480920258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB7FA71A575, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:26:58.000735894Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:26:58.018951785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:26:58.018975375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:26:58.01899455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=error ts=2026-07-02T23:27:02.248250587Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9CB95235B022, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=warn ts=2026-07-02T23:27:03.019070172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:03.311054999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:27:03.316173422Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB991D6C7D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:27:03.51545674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:27:03.519739575Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CB99DFD23B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:27:07.288313455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:27:07.293282382Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBA7EE63DC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:27:07.506862942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:27:07.509226032Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBA8BC64110, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:27:08.01961779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:09.083416201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:27:09.085650206Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBAE9BE3393, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-02T23:27:13.019541996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:14.451508428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:27:14.453828001Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBC29B69B69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:27:14.733520811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:27:14.735817075Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBC3A84BE37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:27:15.413389115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:27:15.415549501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBC6308E4C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:27:17.650897232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:27:17.653259376Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBCE868D2D1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:27:18.01897236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:18.448941197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:27:18.452731061Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBD180FD14E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:27:18.452750005Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:27:18.452771955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:27:18.459761902Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBD187BACE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:27:18.594775712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:27:18.599221755Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBD20CA64A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:27:19.098608803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:27:19.101006643Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBD3EB28D50, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:27:22.358618108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:27:22.361055224Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBE01025FDB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:27:23.018952193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:23.368480394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:27:23.371579069Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBE3D3E5268, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:27:25.420649883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:27:25.425113902Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBEB7A6435D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:27:28.019399306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:28.019399971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:27:28.01942976Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:27:30.877846487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:27:30.880167581Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CBFFCCB18BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:27:32.578326405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:27:32.580572498Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC06225F087, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:27:32.751109959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:27:32.753346688Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC06C720D6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:27:33.019855586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:33.755690295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:27:33.758262029Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC0A857FFED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:27:33.776286159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:27:33.782509142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC0A9CB8F15, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:27:34.24901976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:27:34.253577468Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC0C5DE9A6F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:27:34.28161373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:27:34.283760191Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC0C7A95EC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:27:35.278559174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:27:35.280734243Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC1031742F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:27:35.784072129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:27:35.786284596Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC12139AB08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:27:36.035277832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:27:36.037579179Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC13032EB08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:27:38.019665243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:39.124704248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:27:39.126898488Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC1E857816B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:27:39.338399383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:27:39.340572271Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC1F5141FF3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:27:39.739602715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:27:39.742147078Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC20D01536F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:27:41.052403432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:27:41.054821585Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC25B403FF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:27:43.019421877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:43.674189842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:27:43.678881628Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC2F7A91F56, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:27:48.018827919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:50.281601615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:27:50.286063751Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC4817AC185, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:27:50.958585819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:27:50.960917981Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC4A9B37EDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:27:53.019704183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:55.089286573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:27:55.093522092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC5A0055CDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:27:55.581548087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:27:55.59152106Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC5BDB2E189, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:27:58.000684448Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:27:58.018915922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:27:58.018937457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:27:58.018964318Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:27:58.565278883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:27:58.570375381Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC66F3D3000, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:27:58.83371046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:27:58.836046081Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC67F171C28, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:28:00.174343989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:28:00.176853792Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC6CF011B48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:28:00.247178058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:28:00.249502809Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC6D3571E20, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:28:03.019353659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:05.949620654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:28:05.952041936Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC8273CC5EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:28:06.688265431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:28:06.690576926Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC85342B589, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:28:07.032554016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:28:07.035664347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC867D3F38A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:28:08.019001445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:08.556928548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:28:08.561961945Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC8C2CD3CB0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:28:11.815784817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:28:11.818272373Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CC984E43206, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:28:13.019270215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:14.613024047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:28:14.615634296Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCA2BA0EF2F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:28:15.621974381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:28:15.624422777Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCA67C15EBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:28:16.723321801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:28:16.72625591Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCAA968C4AD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:28:17.020146792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:28:17.022515128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCABB165605, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:28:18.019349401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:20.060588891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:28:20.06340562Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCB7057B90E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:28:20.2019218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:28:20.208224819Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCB78F89C16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:28:20.516909718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:28:20.519307556Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCB8B841CC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:28:22.737033309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:28:22.739559335Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCC0FDADB9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:28:23.019095021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:28:28.019183386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:28.019198949Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:28:28.019224615Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:28:28.028014998Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-07-02T23:28:28.030035367Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.01008918s
level=info ts=2026-07-02T23:28:30.228328238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:28:30.230673627Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCDCE5D89D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:28:30.412190072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:28:30.414828489Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCDD9539847, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:28:32.380881604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:28:32.383290715Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCE4EAAC3A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:28:33.019658252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:34.534324187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:28:34.536971043Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCECF06BB57, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:28:34.964059624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:28:34.966335692Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCEE8A1673B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:28:37.489888958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:28:37.492587891Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CCF7F326937, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:28:38.019138027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:40.947537778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:28:40.952080066Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD04D678A14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:28:41.191878537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:28:41.198383004Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD05C13D36A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:28:42.197968462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:28:42.200347044Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD097CE865C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:28:42.451899547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:28:42.454177463Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD0A6F08897, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:28:43.019529734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:46.808172741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:28:46.810572454Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD1AA99161B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:28:47.140411988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:28:47.144452516Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD1BE7ED7E8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:28:47.702895104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:28:47.707718677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD1E0135F3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:28:48.019534755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:49.35372775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:28:49.356168591Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD24254412B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:28:50.912587768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:28:50.914789021Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD29F3C9C60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:28:53.019597754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:53.878495924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:28:53.885810134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD35051194B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:28:54.239958542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:28:54.244650219Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD365B58064, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:28:54.607633174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:28:54.610707584Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD37B87007A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:28:55.368563629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:28:55.373130211Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD3A8F879FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:28:58.001179671Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:28:58.019476153Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-07-02T23:28:58.019481853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:28:58.019501386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:28:58.019525822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:28:58.019584266Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-02T23:28:58.019594082Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:28:58.019611046Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.343µs success=true
level=info ts=2026-07-02T23:29:02.154339998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:29:02.156753827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD53D4D976A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:29:03.019191012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:04.842192206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:29:04.844614833Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD5DD83B6A1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:29:05.013252023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:29:05.015498136Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD5E7B452B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:29:05.34825756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:29:05.351145696Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD5FBB32235, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:29:05.391170712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:29:05.395355915Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD5FE579851, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:29:05.908605259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:29:05.910716281Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD61D10CEE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:29:06.540501989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:29:06.542865049Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD642BC7533, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:29:08.01981107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:08.084973382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:29:08.087456034Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD69ECBD8A2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:29:10.516926888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:29:10.519267933Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD72FBF8105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:29:13.019138172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:29:18.019263659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:19.592025562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:29:19.594521951Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CD94CAB26FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:29:23.019806325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:23.738254382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:29:23.740434897Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDA43CB712F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:29:23.740451497Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:29:23.740472764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:29:23.742231475Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDA43E92516, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:29:24.746878719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:29:24.749116685Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDA7FEA7865, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:29:24.757163441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:29:24.759035502Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDA8083A7EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:29:25.224682908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:29:25.2268809Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDA9C65D731, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:29:28.019081231Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:29:28.019111139Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:29:28.019128981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:28.240185718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:29:28.242544679Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDB5023AB49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:29:28.242558814Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:29:28.242577948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:29:28.244661097Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDB5045B822, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:29:28.44609257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:29:28.44832322Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDB5C677E26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:29:30.908726545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:29:30.91140437Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDBEF34BE19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:29:31.528022912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:29:31.532288702Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDC1439BA19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:29:31.767873689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:29:31.774323981Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDC22A5ABFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:29:32.457402665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:29:32.459995862Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDC4B821233, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:29:32.553183589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:29:32.555424976Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDC513413F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:29:33.01966851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:34.648553092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:29:34.650942134Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDCCE1C5A18, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:29:34.662976086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:29:34.665043054Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDCCEF4A5B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:29:38.01882624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:38.592856921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:29:38.596105715Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDDB94160BA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:29:38.596127426Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:29:38.596169424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:29:38.601186183Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDDB991A495, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:29:38.834725724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:29:38.839105171Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDDC7BF5A9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:29:39.167861342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:29:39.170032541Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDDDB786AA8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:29:40.616531871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:29:40.619086191Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDE31D5EFA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:29:41.152490295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:29:41.154908505Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDE51C7AB06, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:29:43.019397734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:43.674773052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:29:43.677128031Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDEE81C9F9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:29:46.307251221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:29:46.311230203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDF851DDA69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:29:46.542741708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:29:46.551839158Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDF93761318, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:29:46.867910768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:29:46.872574285Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CDFA692F406, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:29:48.019665818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:48.987776535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:29:48.990112065Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE024C9B879, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:29:50.295747804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:29:50.298981912Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE072CC9B1F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:29:51.074963995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:29:51.080173378Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE0A15C4A8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:29:53.019749459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:53.162025886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:29:53.164778614Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE11D9E8078, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T23:29:53.164792874Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:29:53.164811527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:29:53.16946946Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE11DE74DED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:29:58.000607445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:29:58.018824328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:29:58.018846957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:29:58.018849497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:29:58.293391652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:29:58.298362184Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE24F9795A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:29:59.56654891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:29:59.569296875Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE29B58C29E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:29:59.569314208Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:29:59.569333206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:29:59.571261387Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE29B7C80F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:30:00.01067927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:30:00.013092239Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE2B5CFC0FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:30:01.176925619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:30:01.179291944Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE2FB528556, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:30:03.019136058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:30:08.019106555Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:08.676141023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:30:08.678989742Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE4BA56223F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:30:13.001597942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:30:13.006587054Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE5BC462D7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:30:13.019615723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:13.862531648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:30:13.865417499Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE5EF774539, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:30:14.620541906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:30:14.623263718Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE61CA139E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:30:14.764376186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:30:14.766588632Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE625318DFF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:30:15.386300155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:30:15.388971011Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE64A459264, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:30:16.929194936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:30:16.931744484Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE6A63B1BD5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:30:17.179440357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:30:17.18162813Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE6B52449ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:30:18.018835144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:18.741187916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:30:18.743621236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE7123C9855, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:30:20.415397881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:30:20.417795612Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE776077739, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:30:20.896100212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:30:20.89834273Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE792ACA3EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:30:21.788622952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:30:21.790963181Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE7C7DFE669, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:30:22.646142322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:30:22.648498979Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE7FAFD9388, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:30:23.019849719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:23.760018911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:30:23.764494694Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE83D815C83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:30:28.019242943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:30:28.019267908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:30:28.019279738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:28.073510294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:30:28.079719333Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE93EB6D82B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:30:28.079736016Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:30:28.079755527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:30:28.081679529Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE93ED72D3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:30:28.406832674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:30:28.409366321Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE9525B09CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:30:28.797300818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:30:28.800564914Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE969AEC048, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:30:28.819680197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:30:28.829917442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE96B6FFC2C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:30:28.8299339Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:30:28.829955342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:30:28.939510317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE971F6EB8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:30:29.703704348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:30:29.708342538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CE99FC94270, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:30:32.397126641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:30:32.399664536Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEA4033A0E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:30:32.853757248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:30:32.85622144Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEA5B6AD7DB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:30:33.019436615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:34.303746445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:30:34.306057844Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEAB1D5E3F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:30:37.93998122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:30:37.943944563Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEB8AAB8E88, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:30:38.019062639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:38.996741868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:30:39.001310128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEBC9B0F416, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:30:40.814467503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:30:40.817346024Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEC35EE3072, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:30:43.001795206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:30:43.006503217Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CECB868C645, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:30:43.019585743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:43.344411526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:30:43.346650048Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CECCCB242B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:30:46.497057279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:30:46.499411293Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CED889D68F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:30:47.303675954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:30:47.305924482Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEDB8B0207E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T23:30:47.305941133Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:30:47.305959102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:30:47.307794883Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEDB8CEB163, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:30:47.913546025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:30:47.920523218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEDDD5276CF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:30:48.019610736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:30:53.019462624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:56.081562554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:30:56.088089638Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEFC41E1D1D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:30:56.088107712Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:30:56.088127626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:30:56.090767919Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEFC4477A7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:30:56.822651688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:30:56.825015012Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CEFF01078E3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:30:57.329347442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:30:57.33222379Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF00E4594FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:30:57.610067817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:30:57.612409173Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF01EFF1FB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:30:58.000415978Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:30:58.019766165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:30:58.019797161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:30:58.019806331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:30:58.945017878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:30:58.947332595Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF06E8F7E40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:30:59.213303107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:30:59.216228756Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF07E919DB8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:31:02.102770606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" idle=1 full=1 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:31:02.105246605Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF12AC984AE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:31:03.019202592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:06.835076133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:31:06.839565802Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF244FA9594, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:31:08.019321007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-07-02T23:31:08.381408175Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9CF2A0E372EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:31:08.898641165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:31:08.902988724Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF2BFF7810F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:31:10.203531492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:31:10.20592773Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF30DA17F04, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:31:12.285022153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:31:12.287410393Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF389B1FAEE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:31:12.287427517Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:12.28744635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:31:12.289351376Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF389D1CF12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-07-02T23:31:13.019616319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:13.657184742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:31:13.659335971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF3DB791047, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:31:14.203171503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:31:14.20550451Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF3FC07152E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:31:14.356747262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:31:14.359072697Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4052D8D77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:31:14.35908516Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:14.359100791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:31:14.361166936Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4054CCC0C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:31:17.167897036Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:31:17.170171019Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4ACBBF24C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:31:17.82491056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:31:17.827101841Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4D3E49EAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:31:18.019391613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:18.069551916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:31:18.071677039Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4E277E3E0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:31:18.071689646Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:18.071708671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:31:18.07344634Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4E295BDB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:31:18.376999828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:31:18.379093133Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF4F4CB43B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:31:19.671555625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:31:19.674720135Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF54203647B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:31:20.260413425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:31:20.267031196Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF56550D6E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:31:20.530694071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:31:20.533083566Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF5752CF738, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:31:21.414542522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:31:21.4174808Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF5A9DC3199, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:31:22.579305146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:31:22.582192614Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF5EF4FF1D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:31:23.019828992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:24.44980481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:31:24.454635066Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF65EEA772A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:31:24.454654159Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:24.454674952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:31:24.456605706Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF65F0B7323, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:31:24.548823142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:31:24.551282849Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF664AA8E17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:31:26.096245041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:31:26.098700959Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF6C0E8DE08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:31:26.579699783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:31:26.586416123Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF6DDFB42FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:31:26.826251595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:31:26.8309236Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF6EC8FB523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:31:28.019691499Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:31:28.019726586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:31:28.019728922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:28.570248886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:31:28.572443434Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF7545C86E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:31:29.369731841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:31:29.373446959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF7841B8BBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:31:30.486593974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:31:30.492196849Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF7C6C3FDE1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:31:32.239944212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:31:32.24248058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF82F1A8170, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:31:33.019326693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:33.414465327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:31:33.417443634Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF8751E1315, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:31:34.214932631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:31:34.217394861Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF8A4D28FD9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:31:38.01938711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:39.934698392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:31:39.937633937Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CF9F9C20CE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:31:42.510391314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:31:42.512750314Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFA9344C6A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:31:43.019828355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:46.153340255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:31:46.155773728Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFB6C68294F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:31:46.155788333Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:46.155806107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:31:46.157757419Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFB6C894DD4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:31:47.671291097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:31:47.674251508Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFBC6E44C9F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:31:48.019036302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:48.068184945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:31:48.07054082Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFBDE899523, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:31:48.070552604Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:48.070569539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:31:48.072532034Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFBDEA9AEBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:31:48.150002241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:31:48.152142082Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFBE3678B7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:31:50.057988621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:31:50.062425261Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFC5543935C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:31:50.062441921Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:50.062463029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:31:50.068995632Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFC55A8C480, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:31:52.448864591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:31:52.454113395Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFCE3CD5532, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-07-02T23:31:53.019380101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:56.359429375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:31:56.362384949Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFDCCBF53AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:31:56.362400724Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:31:56.362419701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:31:56.364834Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFDCCE654BC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:31:58.000687615Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:31:58.018902785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:31:58.018926526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:31:58.01895305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:31:58.5735936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:31:58.575881696Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFE50B3AA1B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:32:00.234561388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:32:00.236950682Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFEB3B4E5A5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:32:00.236965271Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:32:00.236981078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:32:00.23884073Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFEB3D4F019, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:32:02.100859137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:32:02.103227811Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFF22F22D8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:32:03.019349277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:04.061689015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:32:04.06408261Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFF97D1D1E2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:32:04.82707025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:32:04.829844787Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9CFFC57756C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:32:06.564554255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:32:06.569436393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D002D28F85A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:32:06.569453515Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:32:06.569473158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:32:06.571306416Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D002D475BF2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:32:06.59977308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:32:06.6015351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D002F159BB1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:32:06.858130482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="11 MB" avg_comp="3.7 MB" total_uncomp="99 MB" avg_uncomp="33 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:32:06.864543899Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D003EC0C11E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:32:08.019820605Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:08.040966713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:32:08.047361544Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D00853FCBA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:32:08.571336952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:32:08.573525494Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D00A49D8C63, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:32:08.622842825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:32:08.624992065Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D00A7AE4A7F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:32:09.33893272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:32:09.343279897Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D00D27E935A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:32:10.604907088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:32:10.609630246Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D011DF8C178, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:32:10.706845928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:32:10.709022891Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0123E5BC27, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:32:11.489960486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:32:11.493177065Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0152A1C4E7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:32:12.454566912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:32:12.45981907Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D018C3AC9ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:32:12.546993307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:32:12.549057864Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D019192F38F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:32:12.549071374Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:32:12.549088965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:32:12.550953714Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0191B1482E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:32:12.6063199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:32:12.608268403Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D01951B0D24, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:32:13.019701347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:13.418500386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:32:13.420898077Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D01C58858AC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:32:13.974573822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:32:13.977337038Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D01E6AF860B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:32:17.09525347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:32:17.099313635Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D02A0C72D33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:32:17.579362542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:32:17.583842899Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D02BDAB2032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:32:17.795500509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:32:17.797662086Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D02CA6ADB42, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:32:18.018983066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:18.677027913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:32:18.679280851Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D02FEF6287B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:32:18.749495213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:32:18.751609748Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0303464120, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:32:20.074252437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:32:20.076754423Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D035240B07E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:32:23.019375926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:24.944627648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:32:24.946856736Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D04748AF585, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:32:28.01944977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:32:28.019475769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:32:28.019474629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:29.396457499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:32:29.398900474Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D057DE5979B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:32:30.321394542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:32:30.323425113Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D05B50457F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:32:30.361477198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:32:30.363835176Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D05B769294A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:32:33.019674412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:33.569906364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:32:33.5724976Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0676A8B39F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:32:34.240994881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:32:34.243543782Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D069EA85FFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:32:34.666541294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:32:34.668677175Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D06B8025293, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:32:37.383194891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:32:37.385528885Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0759F17F7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:32:38.018870966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:40.069808127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:32:40.074336812Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D07FA352A7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:32:42.493570237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:32:42.498312614Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D088AAF66B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:32:43.019592423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:45.209254032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:32:45.212124129Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D092C70667E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:32:48.019042827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:49.598329399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:32:49.602932295Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0A322839B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:32:50.227652731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:32:50.229807837Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0A5786320F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:32:52.393845848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:32:52.400806839Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0AD8EC5BE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:32:53.019671163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:54.766180809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:32:54.771412853Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0B6636A93A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:32:54.771429287Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:32:54.771447393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:32:54.773399558Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0B665A44F2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:32:55.274945337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:32:55.277243998Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0B845F6C95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:32:57.37888241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:32:57.381994406Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C01D3B668, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:32:57.54946296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:32:57.555722818Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C0C2D1FAD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:32:57.697146697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:32:57.701340213Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C14DCED0A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:32:57.701354766Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:32:57.701371685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:32:57.703101718Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C14F93A7A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:32:57.927384912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:32:57.929573252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C2276C90E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:32:58.000658738Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:32:58.018868841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:32:58.018893075Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:32:58.018929596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:32:58.853250415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:32:58.855502208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0C59A710B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:33:01.29358215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:33:01.29603158Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0CEB1CF943, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:33:01.884464166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:33:01.887011426Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0D0E55B7EA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:33:02.617648852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:33:02.619911383Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0D3A06FD5B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:33:03.019620606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:03.389044992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:33:03.391260721Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0D6800FCDF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:33:05.086585552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:33:05.088847166Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0DCD2FF497, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:33:05.08886421Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:05.08888123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:33:05.090754194Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0DCD506F7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:33:05.933255474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:33:05.935555899Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0DFFA75A55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:33:08.019334627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:09.489888429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:33:09.492107171Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0ED3A55F60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:33:11.897109584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:33:11.899199487Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0F631EC663, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:33:11.899213812Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:11.89923103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:33:11.900926588Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D0F633B10D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:33:13.019728088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:14.97161843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:33:14.973848346Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D101A61AA01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:33:16.089175092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:33:16.092053812Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D105D0580EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:33:17.625069741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:33:17.627079952Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D10B88886D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=warn ts=2026-07-02T23:33:18.019172257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:18.099417187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:33:18.10194824Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D10D4D1FB86, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:33:18.666675352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:33:18.668736777Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D10F69DB676, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:33:19.368565048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:33:19.37085251Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1120764442, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:33:19.370867756Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:19.37088525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:33:19.372607789Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D11209460D7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:33:21.12867513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:33:21.13128647Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D118963DA7E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:33:23.019501533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:23.598086243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:33:23.600316942Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D121C8E5D5A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:33:23.91414359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:33:23.916264501Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D122F6416EF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:33:23.91627665Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:23.916292137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:33:23.918002165Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D122F81532C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:33:25.565253283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:33:25.567279373Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1291CDF9C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:33:26.069708152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:33:26.074443151Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D12B0056648, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:33:26.88905407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:33:26.895525836Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D12E0F5957E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:33:26.895542796Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:26.895561232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:33:26.897927327Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D12E11AD306, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:33:27.229919113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:33:27.232230953Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D12F509DDED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:33:28.018764027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:33:28.018790243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:33:28.018807817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:28.027768838Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-07-02T23:33:28.029809492Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010065971s
level=info ts=2026-07-02T23:33:30.364260866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:33:30.367303159Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D13AFE6EA7D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:33:32.098866379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:33:32.103920206Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D141766DCBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:33:33.019139712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:33.684428709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:33:33.686669134Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1475BF1AA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:33:36.103707793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:33:36.105942997Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1505F34398, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:33:36.195176104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:33:36.197385988Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D150B65C871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:33:38.018983377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:39.705024886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:33:39.712769748Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D15DCED88A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:33:41.47119242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:33:41.47614012Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D164608E7AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:33:43.019789063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:43.68018592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:33:43.682619139Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D16C98C83DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:33:43.69179204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:33:43.69367487Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D16CA39704E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:33:45.562528258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:33:45.564725048Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1739BCE5D5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:33:45.722244576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:33:45.726218641Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D17435C5AA6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:33:45.726233491Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:33:45.726250119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:33:45.732082576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1743B8FA6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:33:46.762766553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:33:46.767366811Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D17816B9A37, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:33:47.960860015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:33:47.963089768Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D17C8B149AB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:33:48.019212893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:50.088446648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:33:50.090755846Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1847827123, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:33:52.166681491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:33:52.168957636Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D18C361D185, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:33:53.018944146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:54.329563685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:33:54.332070816Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D19444F8D12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:33:54.530583131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:33:54.532717171Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1950475605, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:33:54.743247154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:33:54.746534876Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D195D02BD0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:33:56.00185961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:33:56.00849122Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D19A83A7E6E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:33:57.513748491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:33:57.518787333Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1A023F2FF8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:33:58.000391374Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:33:58.018701735Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:33:58.018827953Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-02T23:33:58.018836235Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:33:58.0188471Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:33:58.01885337Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.73µs success=true
level=info ts=2026-07-02T23:33:58.018874984Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:33:58.018912928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:33:59.162257472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:33:59.164437208Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1A645941C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:34:00.856066005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:34:00.858130814Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1AC94D2EFE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:34:01.947143799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:34:01.94931663Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1B0A576920, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:34:02.641488108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:34:02.643672807Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1B33BA5DD2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:34:03.018924607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:04.359623185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:34:04.361858163Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1B9A23208D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:34:04.810058598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:34:04.812524293Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1BB4FF23B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:34:08.019077462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:11.311958869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:34:11.314322433Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1D388917F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:34:12.557635372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:34:12.560288425Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1D82CB145F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:34:13.019546514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:14.057671981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:34:14.060399558Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1DDC35D9E9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:34:14.259922079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:34:14.266332177Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1DE878C9FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:34:15.725737853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:34:15.730266977Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1E3FBDBADA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:34:18.019727137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:19.290452841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:34:19.295238279Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1F143CAE54, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:34:21.614514633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:34:21.616871017Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D1F9E9DCF11, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:34:23.019111416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:25.263197572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:34:25.270076074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D20785C2C95, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:34:27.323627514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:34:27.330036612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D20F3255B12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:34:28.019317011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:28.019324253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:34:28.01935257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:34:32.073991223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:34:32.078658324Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D220E2F95A6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:34:33.018958376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:33.394316656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:34:33.396647183Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D225CBE770B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:34:34.533962772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:34:34.536186773Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D22A0AAD747, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:34:38.018828085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:39.098054814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:34:39.100548579Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D23B0B84AE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:34:40.757946771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:34:40.76011215Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2413A5132E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:34:41.393406329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:34:41.395793762Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2439862929, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:34:41.431840379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:34:41.434083784Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D243BCE0A98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:34:43.019184933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:43.5045979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:34:43.506696092Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D24B75AA537, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:34:43.776937079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:34:43.779029733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D24C7954CE4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:34:45.513909485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:34:45.516091738Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D252F1F2542, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:34:46.362470775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:34:46.365549108Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2561C03037, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:34:46.721500626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:34:46.727531705Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D25775420D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:34:47.636900567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:34:47.639089685Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D25ADA94639, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:34:48.018939304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:48.282713888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:34:48.284960254Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D25D427DDB7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:34:53.019106407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:53.391032509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:34:53.39336978Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2704A413ED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:34:54.584725201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="2.3 kB" avg_comp="2.3 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:34:54.587014159Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D274BC935E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:34:55.612337916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:34:55.61458247Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D278909BA43, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:34:57.20519192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:34:57.207779391Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D27E7FCB36B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:34:58.001142704Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:34:58.019449637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:34:58.019460788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:34:58.019473294Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:35:00.099684517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.9 kB" avg_comp="967 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:35:00.101884696Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D289480B88F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:35:00.12692597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:35:00.129261021Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D289620176A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:35:01.445746556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:35:01.447899665Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D28E4BBD568, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:35:01.772613147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:35:01.77519135Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D28F83D55F5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-07-02T23:35:03.018843762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:04.800015898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:35:04.80531726Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D29ACD79D76, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:35:05.912127194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:35:05.918763971Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D29EF3700B4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:35:08.019791729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:08.973241312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:35:08.978597485Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2AA591FF2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:35:09.235064724Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9D2AB4E28DA7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:35:10.568557851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:35:10.572585748Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2B0498A342, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:35:13.019846839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:15.024513945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:35:15.029400563Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2C0E3E6F81, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:35:15.577776311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:35:15.582195853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2C2F330347, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:35:16.564992207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:35:16.569239932Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2C6A0831AA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:35:18.019516759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:19.172643953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:35:19.175202312Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2D0559F45B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:35:22.805419025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:35:22.807562104Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2DDDDE5769, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:35:23.019292129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:24.011529979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:35:24.013789968Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2E25C302C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:35:24.19734695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:35:24.199558778Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2E30D6565A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:35:28.019511939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:35:28.019518759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:28.019541499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:35:28.873182452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:35:28.87572049Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2F478C8A14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:35:29.089879807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:35:29.092039638Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D2F54737179, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:35:32.874495716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:35:32.876709955Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D303608E578, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:35:33.019058864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:33.931595203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:35:33.933802482Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D30750ADDE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:35:35.694730307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:35:35.697707651Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D30DE2E3ECC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:35:36.213329249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:35:36.21903764Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D30FD4061FB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:35:38.018978386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:38.101306132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:35:38.103557829Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D316D934891, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:35:39.616876536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:35:39.619093354Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D31C7E94852, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:35:41.457665074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:35:41.459911214Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3235A1A295, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:35:42.908510427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:35:42.91070461Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D328C1AD66D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:35:43.01941208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:43.266463459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:35:43.268444646Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D32A16F3C93, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:35:47.788747808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:35:47.791290339Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D33AF04169C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T23:35:47.791308459Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:35:47.791330929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:35:47.797108683Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D33AF5E3092, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:35:47.873181019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" idle=1 full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:35:47.877059469Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D33B421F582, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:35:48.019429317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:48.920189799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:35:48.922837994Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D33F271853D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:35:49.724192504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:35:49.726526708Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D34225C0827, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:35:51.962730788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:35:51.965043917Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D34A7C98A9E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:35:52.527683386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:35:52.529988708Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D34C9758196, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=warn ts=2026-07-02T23:35:53.0198347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:55.507735641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:35:55.510194425Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D357B17A995, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:35:56.728624027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:35:56.731323071Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D35C3DC866C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:35:58.000810738Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:35:58.019058175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:35:58.019072079Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:35:58.019101351Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:36:00.733414392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:36:00.741212569Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D36B2E2C7CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:36:03.019633338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:04.286004222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:36:04.291577525Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D37867F93E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:36:04.382628696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:36:04.38702956Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D378C326035, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:36:05.213516291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:36:05.216006038Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D37BD98D309, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:36:06.058247719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:36:06.062045462Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D37F007C4E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:36:08.019457298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:10.065559321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:36:10.070508821Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D38DEF3810A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:36:10.070525964Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:36:10.070546425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:36:10.072374219Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D38DF13DF0D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:36:10.23595338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:36:10.238363213Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D38E8F50156, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:36:10.23837811Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:36:10.371581602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:36:10.373907906Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D38F109E3B7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:36:10.44619496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:36:10.448192895Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D38F578E49E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:36:12.129940942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:36:12.134360466Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3959F8242F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:36:13.018938985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:16.354749051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:36:16.359328538Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3A55CBC758, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:36:16.359347914Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:36:16.359368813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:36:16.361282177Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3A55EB6E08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-02T23:36:18.019047949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:19.810672044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:36:19.813011615Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3B23A5CB97, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:36:21.660882838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:36:21.663196754Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3B91EDBF83, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:36:22.414491664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:36:22.416800721Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3BBED8A16F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:36:23.019338032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:23.334178064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:36:23.336496035Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3BF5A9AC40, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:36:24.992513485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:36:24.994791355Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3C5882B1D0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:36:27.00957508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:36:27.011921639Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3CD0BD3EF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:36:27.694608323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:36:27.697335868Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3CF993917E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:36:28.018953807Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:36:28.018982276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:36:28.018993305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:28.311237804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:36:28.313375199Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3D1E5039CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:36:28.987924003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:36:28.990442346Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3D46A78C17, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:36:32.041196509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:36:32.044512457Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3DFCB0D9DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:36:32.044529805Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:36:32.044550014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:36:32.051181021Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3DFD1D5B72, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:36:32.796325468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:36:32.801086829Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3E29CE1D19, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:36:33.018946994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:33.725314444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:36:33.727863769Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3E6109E503, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:36:36.54674921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:36:36.549130806Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3F093410E4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:36:36.727716547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:36:36.730331579Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3F13FF581F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:36:37.220846525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:36:37.223089256Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3F3160CAFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:36:37.263139513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:36:37.265109872Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3F33E39902, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:36:38.019249976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:39.1921918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:36:39.194933394Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3FA6E955C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:36:40.505410565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:36:40.510088008Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D3FF54C23CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:36:43.019753809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:43.875583853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:36:43.878396975Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D40BE0C476B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:36:44.688578767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:36:44.691666488Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D40EE8A5BF1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:36:46.931030922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:36:46.935917102Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D41744E7E7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:36:48.019041343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:48.416727113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:36:48.419066097Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D41CCB52C82, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=info ts=2026-07-02T23:36:49.026796235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:36:49.029276879Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D41F1145A26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:36:53.002875577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:36:53.005225327Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D42DE106A8B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=warn ts=2026-07-02T23:36:53.019284991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:54.00188655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:36:54.010349447Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4319F9D9D8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:36:54.010369361Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:36:54.010390496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:36:54.020047349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D431A8FCDC8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:36:54.774710527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:36:54.779424312Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4347CED3BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:36:55.06334988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:36:55.065546466Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4358DF38D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:36:58.000867895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:36:58.01912979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:36:58.01916758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:36:58.019177723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:36:59.696407334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:36:59.69906706Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D446D09D0DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:37:02.774543882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:37:02.776881951Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D45247FFA39, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:37:03.019776517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:03.145116662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:37:03.147446675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D453A955299, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:37:03.217279642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:37:03.219178376Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D453EDF1684, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:37:04.008693975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:37:04.012506273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D456E259B48, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:37:05.342874152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:37:05.352451874Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D45BE03D648, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:37:06.077834075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:37:06.083292787Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D45E987DB51, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:37:06.083306737Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:37:06.08332543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:37:06.088284945Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D45E9DFE72F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:37:08.018851863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:09.31720193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:37:09.319623998Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D46AA798968, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:37:11.080496778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:37:11.084956906Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4713B3B4E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:37:11.661515387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:37:11.666383683Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D473659B903, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:37:13.019395998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:13.988536191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:37:13.991631081Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D47C0F2BBA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:37:18.018992665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:18.166685743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:37:18.171472459Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D48BA14DB70, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:37:18.936321829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:37:18.938481349Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D48E7CEDB14, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:37:19.630539358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:37:19.632822574Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D49113093E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:37:20.349750363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:37:20.35257388Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D493C11C90A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:37:20.352591449Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:37:20.352610933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:37:20.355073963Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D493C38E91B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:37:23.019594015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:23.028736783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:37:23.031440108Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D49DBC3BD5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:37:23.178256233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:37:23.18279496Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D49E4CA5292, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:37:24.205122161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:37:24.207515093Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4A21DE43FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:37:24.615546479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:37:24.617834422Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4A3A539B09, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:37:25.670571734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:37:25.672857772Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4A793553BD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:37:27.555731315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:37:27.558127145Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4AE9940AE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:37:28.019591393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:28.019612727Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:37:28.019646179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:37:28.399201592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:37:28.401458435Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4B1BD8BB7B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=error ts=2026-07-02T23:37:28.401474396Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:37:28.401494212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:37:28.40335879Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4B1BF7A690, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:37:29.505725037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:37:29.508067509Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4B5DCD1BBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:37:31.24208082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:37:31.244893119Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4BC54F89B5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:37:31.511705747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:37:31.514420689Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4BD56087F9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:37:33.018982931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:33.663345894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=2010a74add49f034 immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="147 kB" avg_uncomp="147 kB" full=1 labels="{stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:37:33.665657884Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4C559E118C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-adp\"}"
level=error ts=2026-07-02T23:37:33.665673136Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:37:33.665691147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:37:33.667545992Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4C55BCB7D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:37:35.08942482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:37:35.091815876Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4CAA9EDCED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:37:37.181509701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:37:37.188591887Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4D279A2E8C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:37:38.019320005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:39.454950669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:37:39.460702988Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4DAF0784C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:37:41.903628805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:37:41.905797704Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4E40C5ECA3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:37:43.019240956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:45.04718285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:37:45.049870179Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4EFC27DFEC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:37:45.263561735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:37:45.265841608Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4F090A9B3E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:37:46.861359119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:37:46.864049011Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4F684C45B1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:37:47.908695998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:37:47.911017621Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4FA6B49060, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:37:48.0188652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:48.190357279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:37:48.192634379Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4FB77E1924, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:37:48.570480916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:37:48.573092924Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4FCE27A659, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:37:48.573106428Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:37:48.573123817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:37:48.575430876Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D4FCE4BB06F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:37:51.536254386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:37:51.53857801Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D507EEC8851, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:37:52.593401171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:37:52.596235968Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D50BDF389F1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:37:53.019182659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:53.133486692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:37:53.136102977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D50DE21BC3F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:37:54.426330146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:37:54.430222377Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D512B476128, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:37:58.000907517Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-07-02T23:37:58.019185084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:37:58.019212626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:37:58.019228016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:38:00.091196823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:38:00.096946568Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D527D07EFC2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:38:01.99853168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:38:02.000866959Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D52EE867FF0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:38:03.019229465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:05.090948923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:38:05.095398124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D53A6F93ADD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:38:05.097363158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:38:05.10188439Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D53A75E4650, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:38:05.673351741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:38:05.67555412Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D53C98DF551, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-07-02T23:38:08.019004596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:08.230578755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:38:08.232830917Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5461FBA6B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:38:08.232846902Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:08.232864802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:38:08.234723394Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D54621A606C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:38:09.128011781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:38:09.131114817Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D549785B269, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:38:11.121017605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:38:11.126084461Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D550E6C1C5C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:38:12.391257192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:38:12.393572452Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5559FAE0FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=error ts=2026-07-02T23:38:12.393588522Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:12.393605307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:38:12.39537913Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D555A1966B0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:38:13.019323841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:13.31625562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:38:13.318525491Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D55911CAFBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:38:13.3185405Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:13.318559898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:38:13.320331966Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D55913B1CB5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:38:13.587803811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:38:13.590443756Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D55A14EEAF7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:38:13.590459205Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:13.618793825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:38:13.621315986Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D55A3255106, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:38:17.400012621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:38:17.402307511Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D568486E3A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:38:17.402324356Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:17.402343477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:38:17.404134051Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5684A4ED7C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:38:17.689765288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:38:17.696290442Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D56960D2334, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:38:18.018821477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:18.252187179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:38:18.257768179Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D56B77F6B31, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:38:19.665115573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:38:19.669581748Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D570BA81329, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:38:19.908425295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:38:19.910605175Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D571A063BC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:38:19.991820874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:38:19.993878912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D571EFDA7C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:38:20.013090067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:38:20.01497184Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5720408DD1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:38:20.018992364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:38:20.020838367Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D57209ACD30, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:38:23.019034207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:23.038222089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:38:23.040675909Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D57D495DE73, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:38:25.106257484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:38:25.108633283Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D584FD9AEDC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:38:26.587473623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:38:26.589850295Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D58A822FD0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:38:28.019111869Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:38:28.019109559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:28.019137164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:38:28.032041237Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-07-02T23:38:28.034338844Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.015392649s
level=info ts=2026-07-02T23:38:28.7759271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:38:28.778318151Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D592A94750E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:38:30.746321313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:38:30.748628929Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D59A00594DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:38:33.019439542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:33.308180483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:38:33.310955278Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5A38BB8162, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:38:35.317653396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:38:35.320058525Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5AB07F40D2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:38:35.430398751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:38:35.432489071Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5AB734B3A4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:38:36.734917603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:38:36.737619871Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5B04FA10A8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:38:36.979188337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:38:36.981916871Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5B138ABDFC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:38:38.019010417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:38.832994157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:38:38.835668034Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5B82085265, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:38:42.544266581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:38:42.548237475Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5C5F54565E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:38:43.019853041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:46.316195156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:38:46.322890499Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D4052BC5F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:38:46.369920755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:38:46.374301304Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D43644590, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:38:46.374317716Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:46.374341398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:38:46.380715699Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D43C7775A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:38:47.711448406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:38:47.716237347Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D935EA307, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:38:47.716256835Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:47.716275286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:38:47.718052453Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D937E5772, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:38:47.828393604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:38:47.830885076Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5D9A3160FD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:38:48.019046335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:38:53.019026424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:53.460923405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:38:53.463378046Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5EE9ED6EBE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:38:53.463397273Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:53.463417468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:38:53.465363552Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5EEA0E95E1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:38:53.571600655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:38:53.573530953Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5EF08114BB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:38:53.573542489Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:38:53.683834548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:38:53.685942014Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5EF733548B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:38:53.764175913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:38:53.766129578Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5EFBFBCAE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:38:54.949284633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:38:54.951674884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5F42A34155, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:38:57.599194225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:38:57.601323019Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D5FE094738E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-07-02T23:38:58.000419529Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:38:58.018703071Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:38:58.018771801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:38:58.018795Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:38:58.018811034Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=warn ts=2026-07-02T23:38:58.018815916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:38:58.018818573Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:38:58.018835993Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.662µs success=true
level=info ts=2026-07-02T23:38:58.228608867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:38:58.230968883Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D600619DE16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:39:00.508917907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:39:00.512025651Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D608E10B463, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:39:01.040474645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:39:01.04582698Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D60ADDF288F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:39:01.720518598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:39:01.722763066Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D60D63A66DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:39:01.861078697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:39:01.863294751Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D60DE9A8A6B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:39:02.349973891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:39:02.352187644Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D60FBBE704F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:39:02.689195255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:39:02.695914763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D61103D0DB9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:39:03.019199689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:03.789797655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:39:03.794556998Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6151B60AC0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:39:03.940827826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:39:03.94312836Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D615A915FD3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:39:05.628865153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:39:05.63126087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D61BF3007AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:39:06.276038961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:39:06.280474834Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D61E5E22753, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:39:08.01978562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:11.817162779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:39:11.821670228Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D63302BAE5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-07-02T23:39:13.01941788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:14.496259214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:39:14.49853179Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D63CFB96B52, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:39:14.971539988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9D63EBEC6AED, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:39:15.190513435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:39:15.193293422Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D63F921709D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:39:16.637188787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:39:16.641516094Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D644F752F16, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:39:16.641534701Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:16.641553419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:39:16.643304157Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D644F92799F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:39:16.824841302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:39:16.827022059Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D645A816871, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=warn ts=2026-07-02T23:39:18.019292542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:18.505980218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:39:18.508163998Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D64BEB68BEB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:39:21.043499873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:39:21.045860469Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6555F8F371, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:39:23.019851138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:24.22534145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:39:24.227770096Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6613A036EE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:39:24.303968896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:39:24.306167903Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D66184D6C26, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:39:24.306179855Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:24.306195905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:39:24.307862666Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D66186A0979, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:39:24.887256961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:39:24.889443596Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D663B127963, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:39:24.889462528Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:24.889481418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:39:24.891488798Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D663B3155D3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:39:26.817401616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:39:26.819488586Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D66AE1DB880, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:39:27.816450134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:39:27.818871567Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D66E9ABAD55, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:39:28.008243855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:39:28.010408729Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D66F517E786, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:39:28.019559129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:28.019574249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:39:28.019591001Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:39:28.772009989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:39:28.774449701Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6722A0F33B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:39:33.019798958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:34.822930633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:39:34.82779989Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D688B70F04A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:39:38.019273405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:38.124591292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:39:38.130658543Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D69504DE635, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:39:38.130673614Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:38.130691745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:39:38.132581972Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D69506EE8D6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:39:39.555030719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:39:39.557353368Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D69A558D8C6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:39:39.928257093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:39:39.931985806Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D69BBACF9C5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:39:40.91184056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:39:40.91622504Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D69F656F2B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:39:42.4370641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:39:42.439228896Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6A511EBC2E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:39:42.480255779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:39:42.482222818Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6A53AF2964, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=warn ts=2026-07-02T23:39:43.019320832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:43.126199199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:39:43.128397351Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6A7A323CBD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:39:44.031571306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:39:44.034573974Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6AB034FEB4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:39:45.838970498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:39:45.843918625Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6B1C0C4CE2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:39:47.389206896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:39:47.391895557Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6B784DB9C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:39:48.019467573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:51.338003221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:39:51.34046054Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6C63A74C49, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:39:51.858423177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="1.9 kB" avg_comp="967 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:39:51.860577259Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6C82AC0DCB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=warn ts=2026-07-02T23:39:53.019870281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:55.626536309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:39:55.628838246Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6D63465770, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:39:56.440070089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:39:56.442335567Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6D93C3F91A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-07-02T23:39:56.442350903Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:56.442368278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:39:56.44421101Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6D93E23C01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:39:56.865335809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:39:56.872619071Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6DAD69604F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:39:57.563634375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:39:57.569454967Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6DD6F10629, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:39:58.000434442Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:39:58.018757002Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:39:58.018786247Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:39:58.018846385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:39:58.543864736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:39:58.546122326Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6E1129371C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:39:58.546137947Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:39:58.546171837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:39:58.548011332Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6E114889E6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:39:59.164986692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:39:59.16777851Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6E36319105, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:40:02.162859955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:40:02.165191558Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6EE8E02B2B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:40:03.019109428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:04.178621908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:40:04.180981256Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6F61065746, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:40:05.432347221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:40:05.434695953Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6FABBFBAA1, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:40:06.105664324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:40:06.107794549Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D6FD3E01F07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=warn ts=2026-07-02T23:40:08.019079515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:08.81190879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:40:08.814254342Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D70752EFE3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:40:09.62433285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:40:09.626739241Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D70A59D66B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:40:10.149269144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:40:10.151849454Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D70C4E892FF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:40:11.851743873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:40:11.854096927Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D712A5FE11F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:40:11.917185935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:40:11.919648384Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D712E49FC07, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:40:12.304705264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:40:12.309215072Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7145819813, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=warn ts=2026-07-02T23:40:13.019738157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:14.141322968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:40:14.143836653Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D71B2DA03F8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:40:14.143854411Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:40:14.143873869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:40:14.146021825Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D71B2FBD867, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:40:15.302932268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:40:15.305133755Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D71F81416F0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:40:16.896484442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:40:16.898889231Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D725711D82A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:40:18.019412492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:40:23.019211229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:24.608918106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:40:24.613369748Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7422E3C0E5, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:40:24.842188797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:40:24.848218437Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7430E05CD0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:40:26.024688126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:40:26.027787982Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D747733C81A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:40:27.597193499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:40:27.602315855Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D74D509181F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:40:28.019581284Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:40:28.019605796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:40:28.019620965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:29.605863475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:40:29.608275003Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D754C9C1642, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:40:32.193470316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:40:32.196322884Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D75E6DBF555, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-07-02T23:40:33.019476818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:33.840941043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:40:33.843347212Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D76490994A0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:40:35.510997014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:40:35.513377008Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D76AC9557DD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:40:36.126864414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:40:36.129114025Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D76D1490736, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:40:38.000590016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:40:38.007293007Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D77413BA23B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:40:38.019448895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:38.415051129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:40:38.420806315Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7759DF2440, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:40:38.528995376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:40:38.532600735Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D77608B78FC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:40:38.532612275Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:40:38.532630091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:40:38.538210888Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7760E0ECD8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:40:38.800510753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:40:38.80475688Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7770C4AE05, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:40:39.947717169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:40:39.950102885Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D77B5085376, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=warn ts=2026-07-02T23:40:43.018993523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:45.886459849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:40:45.888835973Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7917026D1A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:40:46.004536596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:40:46.006717601Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D791E0AD235, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:40:46.232848089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:40:46.23540279Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D792BA7494A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:40:47.624183055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:40:47.626533247Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D797E96709D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:40:48.019185903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:48.960655113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:40:48.963082879Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D79CE3F66CA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:40:51.068316315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:40:51.072304263Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7A4BF724A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:40:51.355986469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:40:51.360522435Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7A5D26262A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:40:52.080572303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:40:52.082779825Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7A88329964, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:40:53.019706297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:56.560167449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:40:56.562835085Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7B9333185D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:40:58.000858371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:40:58.019081295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:40:58.019084349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:40:58.019105818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:40:58.682436596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:40:58.684761899Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7C11B3E1CB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:41:00.304190691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:41:00.306841643Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7C7260B21E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:41:00.529301634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="691 B" avg_comp="691 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:41:00.531521733Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7C7FC9305B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-07-02T23:41:03.019539056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:04.628079951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:41:04.630822662Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7D741D8F66, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:41:04.75393721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:41:04.759829518Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7D7BC9B472, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:41:05.193139437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:41:05.195507041Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7D95C695F4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:41:06.933046225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:41:06.935417141Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7DFD7A789C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=warn ts=2026-07-02T23:41:08.019461325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-07-02T23:41:13.019401979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:14.225191118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:41:14.228116356Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D7FB022C798, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:41:18.01972863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:18.510183267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:41:18.517212052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D80AFCEFA98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:41:19.051185271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:41:19.057905813Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D80D009CB41, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:41:19.427656479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:41:19.432457616Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D80E6586816, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:41:21.765561011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:41:21.76787425Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D81718FCB5D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:41:22.077839087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:41:22.079935487Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D81842A2D60, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-07-02T23:41:22.452913469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:41:22.455873398Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D819A92E1A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:41:23.019188068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:24.579058595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:41:24.583569011Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8219646010, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=info ts=2026-07-02T23:41:28.018762648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:41:28.018800569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:41:28.018857528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:30.14719328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:41:30.149671105Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8365281DBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:41:30.312281483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:41:30.314603851Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D836EFC6EA9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:41:30.532290633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:41:30.536219655Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D837C3231C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-07-02T23:41:33.019324248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:33.038460251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:41:33.04306238Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D84119F0206, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=info ts=2026-07-02T23:41:35.195208352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:41:35.197861038Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D84920C65FE, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:41:36.032016572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:41:36.034321411Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D84C3E8DD78, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:41:36.799476699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:41:36.801552531Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D84F1A5B48B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=warn ts=2026-07-02T23:41:38.01880431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:41.967433554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:41:41.969616554Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8625B043DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:41:42.311687319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:41:42.314119182Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D863A36E2DC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:41:43.019396713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:43.240102047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:41:43.242432036Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D86718C473C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:41:44.175037842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:41:44.177296641Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D86A94578D9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:41:44.936329395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:41:44.939025711Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D86D6AC09CC, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:41:46.998518134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:41:47.004211583Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8751C4FE12, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:41:48.019037907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:49.493188382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:41:49.495459724Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D87E641ECFA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:41:50.000900265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:41:50.003101184Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8804840032, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:41:50.21435363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:41:50.216513209Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D88113CD52D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:41:53.019756908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:41:53.412538062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:41:53.41517149Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D88CFE08806, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:41:53.708033004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:41:53.710491527Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D88E17C3E69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:41:56.364969143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:41:56.367265581Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D897FD92A98, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:41:56.755249039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:41:56.757396494Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D89971B51B6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:41:58.000313518Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:41:58.019663065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:41:58.019685207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:41:58.019712242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:01.372163638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:42:01.376540738Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8AAA6D2825, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-07-02T23:42:01.650413211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:42:01.655569996Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8ABB0B4708, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:42:02.68325888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:42:02.685574166Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8AF873E99F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:42:03.019190534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:06.050193782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:42:06.056771503Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8BC164D2DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:42:08.019354374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:09.053186878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:42:09.058020317Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8C74446CFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:42:13.019264589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:14.530951068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:42:14.533295886Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8DBAA1A087, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:42:14.634646022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:42:14.636783376Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8DC0CE075C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:42:16.614378601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:42:16.617422346Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8E36DB5B69, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:42:18.019665623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:20.153825462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:42:20.158914828Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8F09EFEBE6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:42:22.449313433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:42:22.451757046Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8F929AFF94, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-07-02T23:42:23.019027591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:23.151185643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:42:23.153913147Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8FBC72D442, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:42:23.242681299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b94b9dc89b8712c5 immediate=false num_chunks=1 total_comp="18 kB" avg_comp="18 kB" total_uncomp="83 kB" avg_uncomp="83 kB" full=1 labels="{stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:42:23.244817701Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8FC1E1A9B9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"tempo-operator\"}"
level=error ts=2026-07-02T23:42:23.244830476Z caller=flush.go:240 component=ingester loop=5 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:42:23.244846936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:42:23.246673662Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D8FC1FFF8A9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:42:26.695319357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:42:26.699688761Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D908FCCA9B3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:42:28.019574324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-07-02T23:42:28.019578715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:28.019603261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-07-02T23:42:28.663339497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:42:28.668208994Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9105212D77, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:42:28.668224912Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:42:28.668245943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:42:28.670205052Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D910543E104, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:42:31.921251469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:42:31.923581034Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D91C72CD10B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:42:32.596203362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:42:32.598557223Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D91EF674894, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:42:33.019192483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:33.447744278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:42:33.450032866Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9222284794, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-07-02T23:42:33.591238204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:42:33.593426729Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D922AB470C2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:42:35.004829243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:42:35.007027715Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D927EF6BF6C, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:42:37.645326056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="536 B" avg_comp="536 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:42:37.647619374Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D931C5AEDE0, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-07-02T23:42:38.019597394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:38.548720836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:42:38.55103109Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D93523272CD, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-07-02T23:42:38.618001438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:42:38.619992029Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D935651001E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:42:42.367700843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:42:42.370093786Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9435D53412, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=warn ts=2026-07-02T23:42:43.019729733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:44.028414191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:42:44.031013156Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9498D1B102, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:42:44.873144785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:42:44.876005831Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D94CB322119, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-07-02T23:42:44.876024644Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:42:44.876044502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:42:44.88157903Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D94CB89CAA4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:42:45.932567293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:42:45.937633662Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D950A749978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:42:46.13841326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:42:46.143482496Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9516BD7C0B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:42:48.019505327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:48.701109719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:42:48.705723349Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D95AF765CB6, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:42:49.225191235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:42:49.227410967Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D95CE8FB261, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=info ts=2026-07-02T23:42:50.295193961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:42:50.297729095Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D960E58FB23, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:42:50.789812199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:42:50.792477562Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D962BD5F9DF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=warn ts=2026-07-02T23:42:53.019193881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:55.607480629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:42:55.609731213Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D974AFA624B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:42:55.763965058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:42:55.770225479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D97548D4559, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:42:55.770240301Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:42:55.770260875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:42:55.778666079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D97550D299D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:42:57.364039177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:42:57.368467917Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D97B3CED29F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:42:58.000757706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:42:58.018954873Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:42:58.018980275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:42:58.019002174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:42:58.236074866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:42:58.238201327Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D97E7A5E0C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:43:01.934596015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:43:01.937342646Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D98C421826B, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:43:02.154003538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:43:02.15862148Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D98D151B8C3, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:43:03.019785019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:05.934625641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ceccfd6a4958f465 immediate=false num_chunks=1 total_comp="1.2 kB" avg_comp="1.2 kB" total_uncomp="4.2 kB" avg_uncomp="4.2 kB" full=1 labels="{stream=\"openshift-ingress-canary\"}"
level=error ts=2026-07-02T23:43:05.936902707Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D99B286DC3D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-ingress-canary\"}"
level=info ts=2026-07-02T23:43:06.828246517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:43:06.830661188Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D99E7CAD978, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:43:06.830675392Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:43:06.830692741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:43:06.832542673Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D99E7EB1873, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=warn ts=2026-07-02T23:43:08.018946803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:08.359353111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:43:08.362369518Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9A43179511, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:43:08.36238766Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:43:08.362407681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:43:08.367494885Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9A43683793, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:43:09.559376332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=1 total_comp="68 kB" avg_comp="68 kB" total_uncomp="494 kB" avg_uncomp="494 kB" full=1 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-07-02T23:43:09.563826599Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9A8AB38CAB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-07-02T23:43:09.931523547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:43:09.933967271Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9AA0C45C9D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:43:11.658003818Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 404, RequestID: 18BE9D9B0789B25D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist"
level=info ts=2026-07-02T23:43:12.592815632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:43:12.600170558Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9B3FAFB6DA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-07-02T23:43:13.019081599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:16.624189298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=06bfb044b9a75b5b immediate=false num_chunks=1 total_comp="17 kB" avg_comp="17 kB" total_uncomp="102 kB" avg_uncomp="102 kB" full=1 labels="{stream=\"cert-manager\"}"
level=error ts=2026-07-02T23:43:16.628805435Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9C2FCF79A7, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"cert-manager\"}"
level=warn ts=2026-07-02T23:43:18.019326324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:19.178263808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:43:19.180463471Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9CC7E80E33, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"
level=info ts=2026-07-02T23:43:19.805390396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="157 kB" avg_comp="157 kB" total_uncomp="782 kB" avg_uncomp="782 kB" full=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-07-02T23:43:19.808119427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9CED4C5BA2, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-07-02T23:43:20.53499962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:43:20.538166438Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9D18D3CC08, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-07-02T23:43:21.01453183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.4 kB" avg_comp="1.4 kB" total_uncomp="3.1 kB" avg_uncomp="3.1 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-07-02T23:43:21.01934016Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9D357E6FFB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-07-02T23:43:23.019028602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:25.311997828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="1.4 MB" avg_comp="693 kB" total_uncomp="7.8 MB" avg_uncomp="3.9 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-07-02T23:43:25.315721303Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9E3597F237, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-07-02T23:43:25.752748113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ea7c78ab76400a8c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="86 kB" avg_uncomp="86 kB" full=1 labels="{stream=\"image-controller\"}"
level=error ts=2026-07-02T23:43:25.757429Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9E4FEBFCAA, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"image-controller\"}"
level=info ts=2026-07-02T23:43:26.608043288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:43:26.610959326Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9E82C7BF8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:43:28.019120586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:43:28.019162854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:43:28.019181396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:28.026316151Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-07-02T23:43:28.028547428Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009582443s
level=info ts=2026-07-02T23:43:28.968427388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9e4c80b94d101373 immediate=false num_chunks=1 total_comp="99 kB" avg_comp="99 kB" total_uncomp="698 kB" avg_uncomp="698 kB" full=1 labels="{stream=\"appstudio-registry-exporter\"}"
level=error ts=2026-07-02T23:43:28.971102014Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9F0F74EA5E, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-registry-exporter\"}"
level=info ts=2026-07-02T23:43:30.679948525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1d773d0c3eb218e7 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="6.7 kB" avg_uncomp="6.7 kB" full=1 labels="{stream=\"appstudio-cardinality-exporter\"}"
level=error ts=2026-07-02T23:43:30.682438742Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9F7577BEE8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-cardinality-exporter\"}"
level=info ts=2026-07-02T23:43:31.849240282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=122c7b55617f1a36 immediate=false num_chunks=1 total_comp="4.5 kB" avg_comp="4.5 kB" total_uncomp="18 kB" avg_uncomp="18 kB" full=1 labels="{stream=\"openshift-cluster-node-tuning-operator\"}"
level=error ts=2026-07-02T23:43:31.851529014Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9FBB284BE9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-cluster-node-tuning-operator\"}"
level=info ts=2026-07-02T23:43:32.161350964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="349 kB" avg_comp="349 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-07-02T23:43:32.164081685Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9FCDC9451F, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-07-02T23:43:32.457942039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=2 total_comp="2.4 kB" avg_comp="1.2 kB" total_uncomp="12 kB" avg_uncomp="6.2 kB" full=2 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-07-02T23:43:32.463114495Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9FDF9B886D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-07-02T23:43:32.628513001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="15 MB" avg_comp="3.7 MB" total_uncomp="134 MB" avg_uncomp="34 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-07-02T23:43:32.635556244Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9FE9E4A221, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-07-02T23:43:32.68560954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-07-02T23:43:32.68993476Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9D9FED220A91, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-07-02T23:43:33.019565812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:34.698533989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="518 B" avg_comp="518 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-07-02T23:43:34.70082126Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA064FB9D46, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-07-02T23:43:36.581259924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feadc5651def38e4 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="333 kB" avg_uncomp="333 kB" full=1 labels="{stream=\"appstudio-monitoring\"}"
level=error ts=2026-07-02T23:43:36.583514035Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA0D532FD38, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"appstudio-monitoring\"}"
level=warn ts=2026-07-02T23:43:38.019737476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:41.461544284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=daec03023a92ea81 immediate=false num_chunks=2 total_comp="2.0 kB" avg_comp="976 B" total_uncomp="8.2 kB" avg_uncomp="4.1 kB" full=2 labels="{stream=\"crossplane-system\"}"
level=error ts=2026-07-02T23:43:41.463795211Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA1F8170798, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 2, labels: {stream=\"crossplane-system\"}"
level=info ts=2026-07-02T23:43:41.876899273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:43:41.87953613Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA210DA0E62, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:43:41.879549399Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-07-02T23:43:41.879566431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:43:41.881870887Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA210FF8C8D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:43:42.126729921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f47db96eb1979ea3 immediate=false num_chunks=1 total_comp="4.0 kB" avg_comp="4.0 kB" total_uncomp="24 kB" avg_uncomp="24 kB" full=1 labels="{stream=\"release-service\"}"
level=error ts=2026-07-02T23:43:42.128767166Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA21FBA4107, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"release-service\"}"
level=info ts=2026-07-02T23:43:42.926358655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-07-02T23:43:42.930675173Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA24F857D01, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-07-02T23:43:43.019810724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:43.095185501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=977b69f08ed688a0 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="195 kB" avg_uncomp="195 kB" full=1 labels="{stream=\"dummy-service\"}"
level=error ts=2026-07-02T23:43:43.100306753Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA259A367AF, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"dummy-service\"}"
level=info ts=2026-07-02T23:43:43.996187491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="709 B" avg_comp="709 B" total_uncomp="3.0 kB" avg_uncomp="3.0 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-07-02T23:43:43.998367775Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA28F2A174D, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-07-02T23:43:45.965892146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="626 kB" avg_comp="626 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" full=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-07-02T23:43:45.968835535Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA3049DE7D4, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-07-02T23:43:48.019813308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:49.409452866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=13386ca728f42a6a immediate=false num_chunks=1 total_comp="102 kB" avg_comp="102 kB" total_uncomp="421 kB" avg_uncomp="421 kB" full=1 labels="{stream=\"chains-e2e-dqdd\"}"
level=error ts=2026-07-02T23:43:49.414995213Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA3D1FF90C8, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"chains-e2e-dqdd\"}"
level=info ts=2026-07-02T23:43:49.626491523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="28 kB" avg_comp="28 kB" total_uncomp="138 kB" avg_uncomp="138 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-07-02T23:43:49.628948296Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA3DEC34D58, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-07-02T23:43:53.019510168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:55.751254005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="69 kB" avg_comp="69 kB" total_uncomp="547 kB" avg_uncomp="547 kB" full=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-07-02T23:43:55.753841129Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA54BD5C6C9, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-07-02T23:43:56.964140728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="138 kB" avg_comp="138 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-07-02T23:43:56.967174306Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA59423DD8A, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-07-02T23:43:57.794525136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c834e053a2a4474f immediate=false num_chunks=1 total_comp="82 kB" avg_comp="82 kB" total_uncomp="913 kB" avg_uncomp="913 kB" full=1 labels="{stream=\"openshift-marketplace\"}"
level=error ts=2026-07-02T23:43:57.79733643Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA5C59EBB85, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"openshift-marketplace\"}"
level=info ts=2026-07-02T23:43:58.001172669Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-07-02T23:43:58.019463216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-07-02T23:43:58.019482505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-07-02T23:43:58.019484578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-07-02T23:43:58.019482085Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-07-02T23:43:58.019602269Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-07-02T23:43:58.019610799Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-07-02T23:43:58.019628353Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.977µs success=true
level=info ts=2026-07-02T23:44:01.29395586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=eab710ab81744b58 immediate=false num_chunks=1 total_comp="36 kB" avg_comp="36 kB" total_uncomp="238 kB" avg_uncomp="238 kB" full=1 labels="{stream=\"knative-eventing\"}"
level=error ts=2026-07-02T23:44:01.296549542Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 18BE9DA696349CBB, HostID: dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8, api error NoSuchBucket: The specified bucket does not exist, num_chunks: 1, labels: {stream=\"knative-eventing\"}"